00010000:00010000:1.0:1788201705.324242:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450920448 450924543) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324250:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324255:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324258:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.324263:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450920448, 450924544) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.324266:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110088 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.324269:0:17463:0:(rw26.c:898:ll_write_end()) pos 450920448, len 4096, copied 4096 00000080:00200000:1.0:1788201705.324272:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000061448da0[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110088] 00000080:00200000:1.0:1788201705.324274:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000042c5441d fffffc0001029 2:0 ffff93c0c10f07e0 110088 lru 00000080:00200000:1.0:1788201705.324277:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f1b362f4 110088: 1< 2 - - > 2< 450920448 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61673 - -> 00000080:00200000:1.0:1788201705.324279:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000061448da0 00000080:00200000:1.0:1788201705.324280:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.324281:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.324286:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.324294:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.324296:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.324297:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.324303:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.324304:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.324306:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.324308:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324313:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.324318:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.324319:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.324322:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.324324:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110088, 110088] 00000080:00200000:1.0:1788201705.324330:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.324332:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.324349:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.324352:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.324361:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450924544, count: 4096 00000080:00200000:1.0:1788201705.324367:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450924544, bytes: 4096 00000080:00200000:1.0:1788201705.324369:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.324371:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.324372:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450924544, 450928640) 0 0 00000080:00200000:1.0:1788201705.324373:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.324376:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110089, 110089] 00020000:00200000:1.0:1788201705.324378:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450924544 chunk: [450924544, 450928640) 450928640, 4096 00020000:00200000:1.0:1788201705.324379:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.324386:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110089, 110089] 00000008:00010000:1.0:1788201705.324389:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.324390:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.324395:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324404:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450924544 450928639) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324409:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324413:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324417:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.324421:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450924544, 450928640) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.324424:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110089 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.324427:0:17463:0:(rw26.c:898:ll_write_end()) pos 450924544, len 4096, copied 4096 00000080:00200000:1.0:1788201705.324429:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007b33d6c8[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110089] 00000080:00200000:1.0:1788201705.324431:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000948bfa53 fffffc0001029 2:0 ffff93c0c10f02a0 110089 lru 00000080:00200000:1.0:1788201705.324435:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003bca93bb 110089: 1< 2 - - > 2< 450924544 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61674 - -> 00000080:00200000:1.0:1788201705.324436:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007b33d6c8 00000080:00200000:1.0:1788201705.324437:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.324438:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.324443:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.324450:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.324452:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.324454:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.324459:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.324461:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.324463:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.324465:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324469:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.324474:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.324475:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.324478:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.324479:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110089, 110089] 00000080:00200000:1.0:1788201705.324486:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.324488:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.324505:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.324507:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.324517:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450928640, count: 4096 00000080:00200000:1.0:1788201705.324523:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450928640, bytes: 4096 00000080:00200000:1.0:1788201705.324525:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.324527:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.324528:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450928640, 450932736) 0 0 00000080:00200000:1.0:1788201705.324529:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.324532:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110090, 110090] 00020000:00200000:1.0:1788201705.324534:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450928640 chunk: [450928640, 450932736) 450932736, 4096 00020000:00200000:1.0:1788201705.324535:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.324542:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110090, 110090] 00000008:00010000:1.0:1788201705.324545:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.324546:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.324551:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324556:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450928640 450932735) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324562:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324566:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324570:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.324574:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450928640, 450932736) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.324577:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110090 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.324580:0:17463:0:(rw26.c:898:ll_write_end()) pos 450928640, len 4096, copied 4096 00000080:00200000:1.0:1788201705.324582:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000095873f17[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110090] 00000080:00200000:1.0:1788201705.324584:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000072d12bbb fffffc0001029 2:0 ffff93c0c10f0e00 110090 lru 00000080:00200000:1.0:1788201705.324587:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d0854057 110090: 1< 2 - - > 2< 450928640 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61675 - -> 00000080:00200000:1.0:1788201705.324589:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000095873f17 00000080:00200000:1.0:1788201705.324589:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.324591:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.324596:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.324603:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.324605:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.324606:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.324612:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.324614:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.324615:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.324617:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324622:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.324626:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.324628:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.324630:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.324632:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110090, 110090] 00000080:00200000:1.0:1788201705.324639:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.324641:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.324658:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.324660:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.324669:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450932736, count: 4096 00000080:00200000:1.0:1788201705.324675:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450932736, bytes: 4096 00000080:00200000:1.0:1788201705.324677:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.324679:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.324680:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450932736, 450936832) 0 0 00000080:00200000:1.0:1788201705.324682:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.324685:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110091, 110091] 00020000:00200000:1.0:1788201705.324686:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450932736 chunk: [450932736, 450936832) 450936832, 4096 00020000:00200000:1.0:1788201705.324688:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.324694:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110091, 110091] 00000008:00010000:1.0:1788201705.324697:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.324699:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.324704:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324709:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450932736 450936831) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324714:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324719:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324723:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.324727:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450932736, 450936832) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.324729:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110091 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.324733:0:17463:0:(rw26.c:898:ll_write_end()) pos 450932736, len 4096, copied 4096 00000080:00200000:1.0:1788201705.324735:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001ac9f463[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110091] 00000080:00200000:1.0:1788201705.324736:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000071cc4e93 fffffc0001029 2:0 ffff93c0c10f01c0 110091 lru 00000080:00200000:1.0:1788201705.324740:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000278f28f2 110091: 1< 2 - - > 2< 450932736 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61676 - -> 00000080:00200000:1.0:1788201705.324741:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001ac9f463 00000080:00200000:1.0:1788201705.324741:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.324743:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.324748:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.324758:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.324760:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.324761:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.324767:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.324769:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.324770:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.324773:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324777:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.324782:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.324783:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.324785:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.324787:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110091, 110091] 00000080:00200000:1.0:1788201705.324794:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.324796:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.324813:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.324815:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.324825:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450936832, count: 4096 00000080:00200000:1.0:1788201705.324843:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450936832, bytes: 4096 00000080:00200000:1.0:1788201705.324845:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.324847:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.324848:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450936832, 450940928) 0 0 00000080:00200000:1.0:1788201705.324850:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.324853:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110092, 110092] 00020000:00200000:1.0:1788201705.324855:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450936832 chunk: [450936832, 450940928) 450940928, 4096 00020000:00200000:1.0:1788201705.324857:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.324863:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110092, 110092] 00000008:00010000:1.0:1788201705.324866:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.324868:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.324872:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324878:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450936832 450940927) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324883:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324888:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324891:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.324896:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450936832, 450940928) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.324899:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110092 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.324902:0:17463:0:(rw26.c:898:ll_write_end()) pos 450936832, len 4096, copied 4096 00000080:00200000:1.0:1788201705.324905:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002dee0b57[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110092] 00000080:00200000:1.0:1788201705.324906:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004a5801cb fffffc0001029 2:0 ffff93c0c10f0620 110092 lru 00000080:00200000:1.0:1788201705.324910:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000036f50d9f 110092: 1< 2 - - > 2< 450936832 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61677 - -> 00000080:00200000:1.0:1788201705.324911:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002dee0b57 00000080:00200000:1.0:1788201705.324911:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.324913:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.324918:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.324926:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.324928:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.324929:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.324935:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.324937:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.324938:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.324941:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.324946:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.324950:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.324951:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.324954:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.324955:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110092, 110092] 00000080:00200000:1.0:1788201705.324962:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.324964:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.324981:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.324984:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.324993:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450940928, count: 4096 00000080:00200000:1.0:1788201705.324999:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450940928, bytes: 4096 00000080:00200000:1.0:1788201705.325001:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.325003:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.325004:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450940928, 450945024) 0 0 00000080:00200000:1.0:1788201705.325006:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.325009:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110093, 110093] 00020000:00200000:1.0:1788201705.325011:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450940928 chunk: [450940928, 450945024) 450945024, 4096 00020000:00200000:1.0:1788201705.325013:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.325022:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110093, 110093] 00000008:00010000:1.0:1788201705.325027:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.325029:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.325037:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325056:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450940928 450945023) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325066:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325073:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325078:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.325083:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450940928, 450945024) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.325087:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110093 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.325090:0:17463:0:(rw26.c:898:ll_write_end()) pos 450940928, len 4096, copied 4096 00000080:00200000:1.0:1788201705.325093:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000483b8750[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110093] 00000080:00200000:1.0:1788201705.325094:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007d6bddaf fffffc0001029 2:0 ffff93c0c10f0d20 110093 lru 00000080:00200000:1.0:1788201705.325098:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003764cb64 110093: 1< 2 - - > 2< 450940928 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61678 - -> 00000080:00200000:1.0:1788201705.325101:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000483b8750 00000080:00200000:1.0:1788201705.325101:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.325103:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.325109:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.325117:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.325119:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.325120:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.325126:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.325128:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.325129:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.325132:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325137:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.325142:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.325143:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.325145:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.325147:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110093, 110093] 00000080:00200000:1.0:1788201705.325154:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.325156:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.325174:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.325177:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.325187:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450945024, count: 4096 00000080:00200000:1.0:1788201705.325192:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450945024, bytes: 4096 00000080:00200000:1.0:1788201705.325195:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.325197:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.325198:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450945024, 450949120) 0 0 00000080:00200000:1.0:1788201705.325199:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.325203:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110094, 110094] 00020000:00200000:1.0:1788201705.325204:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450945024 chunk: [450945024, 450949120) 450949120, 4096 00020000:00200000:1.0:1788201705.325206:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.325212:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110094, 110094] 00000008:00010000:1.0:1788201705.325215:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.325217:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.325222:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325227:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450945024 450949119) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325233:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325237:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325241:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.325245:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450945024, 450949120) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.325247:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110094 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.325251:0:17463:0:(rw26.c:898:ll_write_end()) pos 450945024, len 4096, copied 4096 00000080:00200000:1.0:1788201705.325253:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ba62b034[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110094] 00000080:00200000:1.0:1788201705.325255:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bb6d95e8 fffffc0001029 2:0 ffff93c0c10f09a0 110094 lru 00000080:00200000:1.0:1788201705.325259:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000065a68ff4 110094: 1< 2 - - > 2< 450945024 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61679 - -> 00000080:00200000:1.0:1788201705.325260:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ba62b034 00000080:00200000:1.0:1788201705.325261:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.325262:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.325267:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.325275:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.325276:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.325278:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.325284:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.325285:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.325287:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.325289:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325294:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.325299:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.325300:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.325302:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.325304:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110094, 110094] 00000080:00200000:1.0:1788201705.325311:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.325312:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.325330:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.325332:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.325342:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450949120, count: 4096 00000080:00200000:1.0:1788201705.325348:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450949120, bytes: 4096 00000080:00200000:1.0:1788201705.325350:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.325352:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.325353:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450949120, 450953216) 0 0 00000080:00200000:1.0:1788201705.325355:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.325358:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110095, 110095] 00020000:00200000:1.0:1788201705.325360:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450949120 chunk: [450949120, 450953216) 450953216, 4096 00020000:00200000:1.0:1788201705.325361:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.325367:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110095, 110095] 00000008:00010000:1.0:1788201705.325370:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.325372:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.325376:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325382:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450949120 450953215) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325388:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325392:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325396:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.325400:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450949120, 450953216) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.325402:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110095 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.325406:0:17463:0:(rw26.c:898:ll_write_end()) pos 450949120, len 4096, copied 4096 00000080:00200000:1.0:1788201705.325408:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fd407155[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110095] 00000080:00200000:1.0:1788201705.325410:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000034fa5c0d fffffc0001029 2:0 ffff93c0c10f0b60 110095 lru 00000080:00200000:1.0:1788201705.325414:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006fa7bdcc 110095: 1< 2 - - > 2< 450949120 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61680 - -> 00000080:00200000:1.0:1788201705.325415:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fd407155 00000080:00200000:1.0:1788201705.325415:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.325417:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.325422:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.325429:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.325430:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.325432:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.325437:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.325439:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.325440:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.325443:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325447:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.325452:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.325454:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.325456:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.325458:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110095, 110095] 00000080:00200000:1.0:1788201705.325464:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.325466:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.325484:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.325486:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.325496:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450953216, count: 4096 00000080:00200000:1.0:1788201705.325502:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450953216, bytes: 4096 00000080:00200000:1.0:1788201705.325503:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.325506:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.325507:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450953216, 450957312) 0 0 00000080:00200000:1.0:1788201705.325508:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.325511:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110096, 110096] 00020000:00200000:1.0:1788201705.325513:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450953216 chunk: [450953216, 450957312) 450957312, 4096 00020000:00200000:1.0:1788201705.325515:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.325521:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110096, 110096] 00000008:00010000:1.0:1788201705.325523:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.325525:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.325530:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325535:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450953216 450957311) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325541:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325545:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325549:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.325553:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450953216, 450957312) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.325556:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110096 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.325560:0:17463:0:(rw26.c:898:ll_write_end()) pos 450953216, len 4096, copied 4096 00000080:00200000:1.0:1788201705.325562:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000857efe87[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110096] 00000080:00200000:1.0:1788201705.325564:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000099fb53c9 fffffc0001029 2:0 ffff93c0c10f0c40 110096 lru 00000080:00200000:1.0:1788201705.325568:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b51277c1 110096: 1< 2 - - > 2< 450953216 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61681 - -> 00000080:00200000:1.0:1788201705.325569:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000857efe87 00000080:00200000:1.0:1788201705.325570:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.325571:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.325576:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.325584:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.325585:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.325586:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.325592:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.325594:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.325595:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.325597:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325602:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.325607:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.325608:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.325611:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.325612:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110096, 110096] 00000080:00200000:1.0:1788201705.325619:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.325621:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.325638:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.325641:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.325650:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450957312, count: 4096 00000080:00200000:1.0:1788201705.325656:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450957312, bytes: 4096 00000080:00200000:1.0:1788201705.325658:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.325660:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.325661:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450957312, 450961408) 0 0 00000080:00200000:1.0:1788201705.325663:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.325666:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110097, 110097] 00020000:00200000:1.0:1788201705.325667:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450957312 chunk: [450957312, 450961408) 450961408, 4096 00020000:00200000:1.0:1788201705.325669:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.325675:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110097, 110097] 00000008:00010000:1.0:1788201705.325678:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.325680:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.325684:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325690:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450957312 450961407) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325695:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325700:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325703:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.325711:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450957312, 450961408) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.325714:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110097 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.325718:0:17463:0:(rw26.c:898:ll_write_end()) pos 450957312, len 4096, copied 4096 00000080:00200000:1.0:1788201705.325720:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000008a7e8a1[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110097] 00000080:00200000:1.0:1788201705.325722:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000150a0261 fffffc0001029 2:0 ffff93c0c10f0540 110097 lru 00000080:00200000:1.0:1788201705.325725:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d888700f 110097: 1< 2 - - > 2< 450957312 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61682 - -> 00000080:00200000:1.0:1788201705.325727:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000008a7e8a1 00000080:00200000:1.0:1788201705.325727:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.325729:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.325734:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.325741:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.325742:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.325744:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.325749:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.325751:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.325753:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.325755:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325760:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.325765:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.325766:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.325769:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.325771:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110097, 110097] 00000080:00200000:1.0:1788201705.325777:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.325779:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.325797:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.325799:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.325809:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450961408, count: 4096 00000080:00200000:1.0:1788201705.325814:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450961408, bytes: 4096 00000080:00200000:1.0:1788201705.325816:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.325818:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.325819:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450961408, 450965504) 0 0 00000080:00200000:1.0:1788201705.325821:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.325824:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110098, 110098] 00020000:00200000:1.0:1788201705.325826:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450961408 chunk: [450961408, 450965504) 450965504, 4096 00020000:00200000:1.0:1788201705.325827:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.325846:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110098, 110098] 00000008:00010000:1.0:1788201705.325849:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.325851:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.325855:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325861:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450961408 450965503) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325867:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325871:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325879:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.325883:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450961408, 450965504) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.325886:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110098 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.325889:0:17463:0:(rw26.c:898:ll_write_end()) pos 450961408, len 4096, copied 4096 00000080:00200000:1.0:1788201705.325891:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a203075b[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110098] 00000080:00200000:1.0:1788201705.325893:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006b693b57 fffffc0001029 2:0 ffff93c0c10f0460 110098 lru 00000080:00200000:1.0:1788201705.325897:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000050829ae2 110098: 1< 2 - - > 2< 450961408 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61683 - -> 00000080:00200000:1.0:1788201705.325898:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a203075b 00000080:00200000:1.0:1788201705.325899:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.325900:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.325905:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.325913:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.325915:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.325916:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.325922:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.325923:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.325925:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.325927:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.325932:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.325940:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.325942:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.325944:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.325946:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110098, 110098] 00000080:00200000:1.0:1788201705.325953:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.325954:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.325972:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.325974:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.325984:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450965504, count: 4096 00000080:00200000:1.0:1788201705.325990:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450965504, bytes: 4096 00000080:00200000:1.0:1788201705.325992:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.325994:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.325995:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450965504, 450969600) 0 0 00000080:00200000:1.0:1788201705.325996:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.325999:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110099, 110099] 00020000:00200000:1.0:1788201705.326001:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450965504 chunk: [450965504, 450969600) 450969600, 4096 00020000:00200000:1.0:1788201705.326003:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.326009:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110099, 110099] 00000008:00010000:1.0:1788201705.326012:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.326014:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.326022:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326030:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450965504 450969599) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326049:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326063:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326070:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.326077:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450965504, 450969600) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.326081:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110099 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.326085:0:17463:0:(rw26.c:898:ll_write_end()) pos 450965504, len 4096, copied 4096 00000080:00200000:1.0:1788201705.326087:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ff1974ff[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110099] 00000080:00200000:1.0:1788201705.326089:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000c8b63b9 fffffc0001029 2:0 ffff93c0c10f00e0 110099 lru 00000080:00200000:1.0:1788201705.326093:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f3ed128a 110099: 1< 2 - - > 2< 450965504 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61684 - -> 00000080:00200000:1.0:1788201705.326094:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ff1974ff 00000080:00200000:1.0:1788201705.326095:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.326096:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.326102:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.326110:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.326112:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.326113:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.326120:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.326121:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.326123:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.326125:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326134:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.326139:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.326140:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.326142:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.326144:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110099, 110099] 00000080:00200000:1.0:1788201705.326151:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.326153:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.326171:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.326174:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.326184:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450969600, count: 4096 00000080:00200000:1.0:1788201705.326190:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450969600, bytes: 4096 00000080:00200000:1.0:1788201705.326192:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.326194:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.326195:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450969600, 450973696) 0 0 00000080:00200000:1.0:1788201705.326197:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.326200:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110100, 110100] 00020000:00200000:1.0:1788201705.326202:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450969600 chunk: [450969600, 450973696) 450973696, 4096 00020000:00200000:1.0:1788201705.326203:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.326215:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110100, 110100] 00000008:00010000:1.0:1788201705.326219:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.326220:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.326225:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326232:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450969600 450973695) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326240:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326245:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326249:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.326253:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450969600, 450973696) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.326256:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110100 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.326260:0:17463:0:(rw26.c:898:ll_write_end()) pos 450969600, len 4096, copied 4096 00000080:00200000:1.0:1788201705.326262:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000092811f7b[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110100] 00000080:00200000:1.0:1788201705.326264:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007e229099 fffffc0001029 2:0 ffff93c0c10f0700 110100 lru 00000080:00200000:1.0:1788201705.326268:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000006c68c5f 110100: 1< 2 - - > 2< 450969600 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61685 - -> 00000080:00200000:1.0:1788201705.326269:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000092811f7b 00000080:00200000:1.0:1788201705.326270:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.326271:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.326277:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.326285:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.326286:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.326287:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.326293:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.326295:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.326297:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.326299:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326304:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.326309:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.326310:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.326313:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.326315:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110100, 110100] 00000080:00200000:1.0:1788201705.326321:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.326323:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.326341:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.326343:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.326353:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450973696, count: 4096 00000080:00200000:1.0:1788201705.326359:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450973696, bytes: 4096 00000080:00200000:1.0:1788201705.326361:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.326363:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.326364:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450973696, 450977792) 0 0 00000080:00200000:1.0:1788201705.326366:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.326369:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110101, 110101] 00020000:00200000:1.0:1788201705.326370:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450973696 chunk: [450973696, 450977792) 450977792, 4096 00020000:00200000:1.0:1788201705.326372:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.326378:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110101, 110101] 00000008:00010000:1.0:1788201705.326381:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.326382:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.326387:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326396:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450973696 450977791) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326401:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326406:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326409:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.326413:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450973696, 450977792) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.326416:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110101 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.326419:0:17463:0:(rw26.c:898:ll_write_end()) pos 450973696, len 4096, copied 4096 00000080:00200000:1.0:1788201705.326422:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000748dc276[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110101] 00000080:00200000:1.0:1788201705.326423:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e26c30b8 fffffc0001029 2:0 ffff93c0c10f0000 110101 lru 00000080:00200000:1.0:1788201705.326427:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000848120bb 110101: 1< 2 - - > 2< 450973696 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61686 - -> 00000080:00200000:1.0:1788201705.326428:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000748dc276 00000080:00200000:1.0:1788201705.326428:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.326429:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.326435:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.326442:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.326444:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.326445:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.326451:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.326453:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.326454:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.326456:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326461:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.326466:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.326467:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.326470:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.326472:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110101, 110101] 00000080:00200000:1.0:1788201705.326478:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.326480:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.326497:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.326499:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.326509:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450977792, count: 4096 00000080:00200000:1.0:1788201705.326515:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450977792, bytes: 4096 00000080:00200000:1.0:1788201705.326517:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.326519:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.326520:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450977792, 450981888) 0 0 00000080:00200000:1.0:1788201705.326521:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.326524:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110102, 110102] 00020000:00200000:1.0:1788201705.326526:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450977792 chunk: [450977792, 450981888) 450981888, 4096 00020000:00200000:1.0:1788201705.326528:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.326534:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110102, 110102] 00000008:00010000:1.0:1788201705.326537:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.326538:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.326543:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326549:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450977792 450981887) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326554:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326558:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326562:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.326567:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450977792, 450981888) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.326570:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110102 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.326573:0:17463:0:(rw26.c:898:ll_write_end()) pos 450977792, len 4096, copied 4096 00000080:00200000:1.0:1788201705.326576:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001c2d38b5[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110102] 00000080:00200000:1.0:1788201705.326578:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cf48f8bb fffffc0001029 2:0 ffff93c0c10f1e00 110102 lru 00000080:00200000:1.0:1788201705.326581:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f4af02c8 110102: 1< 2 - - > 2< 450977792 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61687 - -> 00000080:00200000:1.0:1788201705.326583:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001c2d38b5 00000080:00200000:1.0:1788201705.326583:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.326585:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.326590:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.326597:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.326599:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.326600:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.326606:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.326608:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.326609:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.326612:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326616:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.326621:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.326622:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.326625:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.326627:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110102, 110102] 00000080:00200000:1.0:1788201705.326633:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.326635:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.326653:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.326655:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.326665:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450981888, count: 4096 00000080:00200000:1.0:1788201705.326670:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450981888, bytes: 4096 00000080:00200000:1.0:1788201705.326672:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.326674:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.326675:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450981888, 450985984) 0 0 00000080:00200000:1.0:1788201705.326677:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.326680:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110103, 110103] 00020000:00200000:1.0:1788201705.326682:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450981888 chunk: [450981888, 450985984) 450985984, 4096 00020000:00200000:1.0:1788201705.326683:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.326689:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110103, 110103] 00000008:00010000:1.0:1788201705.326692:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.326694:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.326699:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326704:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450981888 450985983) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326710:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326714:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326718:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.326722:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450981888, 450985984) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.326724:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110103 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.326728:0:17463:0:(rw26.c:898:ll_write_end()) pos 450981888, len 4096, copied 4096 00000080:00200000:1.0:1788201705.326730:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000042b6333d[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110103] 00000080:00200000:1.0:1788201705.326731:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004f5c5b60 fffffc0001029 2:0 ffff93c0c10f11c0 110103 lru 00000080:00200000:1.0:1788201705.326735:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000043b11096 110103: 1< 2 - - > 2< 450981888 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61688 - -> 00000080:00200000:1.0:1788201705.326736:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000042b6333d 00000080:00200000:1.0:1788201705.326736:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.326737:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.326743:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.326753:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.326755:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.326756:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.326762:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.326764:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.326765:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.326767:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326772:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.326776:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.326778:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.326780:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.326782:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110103, 110103] 00000080:00200000:1.0:1788201705.326788:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.326790:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.326808:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.326810:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.326820:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450985984, count: 4096 00000080:00200000:1.0:1788201705.326826:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450985984, bytes: 4096 00000080:00200000:1.0:1788201705.326828:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.326841:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.326843:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450985984, 450990080) 0 0 00000080:00200000:1.0:1788201705.326845:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.326848:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110104, 110104] 00020000:00200000:1.0:1788201705.326850:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450985984 chunk: [450985984, 450990080) 450990080, 4096 00020000:00200000:1.0:1788201705.326851:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.326859:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110104, 110104] 00000008:00010000:1.0:1788201705.326862:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.326863:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.326868:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326873:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450985984 450990079) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326879:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326883:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326887:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.326892:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450985984, 450990080) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.326895:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110104 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.326899:0:17463:0:(rw26.c:898:ll_write_end()) pos 450985984, len 4096, copied 4096 00000080:00200000:1.0:1788201705.326901:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000507e515e[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110104] 00000080:00200000:1.0:1788201705.326903:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000010624e90 fffffc0001029 2:0 ffff93c0c10f1620 110104 lru 00000080:00200000:1.0:1788201705.326907:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000055372077 110104: 1< 2 - - > 2< 450985984 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61689 - -> 00000080:00200000:1.0:1788201705.326908:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000507e515e 00000080:00200000:1.0:1788201705.326909:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.326910:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.326915:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.326923:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.326925:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.326926:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.326932:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.326933:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.326935:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.326937:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.326942:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.326946:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.326947:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.326950:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.326952:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110104, 110104] 00000080:00200000:1.0:1788201705.326958:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.326960:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.326977:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.326979:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.326989:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450990080, count: 4096 00000080:00200000:1.0:1788201705.326995:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450990080, bytes: 4096 00000080:00200000:1.0:1788201705.326997:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.326999:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.327000:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450990080, 450994176) 0 0 00000080:00200000:1.0:1788201705.327002:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.327005:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110105, 110105] 00020000:00200000:1.0:1788201705.327007:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450990080 chunk: [450990080, 450994176) 450994176, 4096 00020000:00200000:1.0:1788201705.327009:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.327017:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110105, 110105] 00000008:00010000:1.0:1788201705.327022:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.327024:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.327032:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327051:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450990080 450994175) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327060:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327068:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327075:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.327080:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450990080, 450994176) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.327084:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110105 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.327087:0:17463:0:(rw26.c:898:ll_write_end()) pos 450990080, len 4096, copied 4096 00000080:00200000:1.0:1788201705.327089:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000087b255b6[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110105] 00000080:00200000:1.0:1788201705.327091:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b8ad1da4 fffffc0001029 2:0 ffff93c0c10f1d20 110105 lru 00000080:00200000:1.0:1788201705.327095:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c8b595a8 110105: 1< 2 - - > 2< 450990080 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61690 - -> 00000080:00200000:1.0:1788201705.327097:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000087b255b6 00000080:00200000:1.0:1788201705.327098:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.327099:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.327106:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.327114:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.327115:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.327117:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.327123:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.327124:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.327126:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.327128:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327133:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.327138:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.327139:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.327142:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.327143:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110105, 110105] 00000080:00200000:1.0:1788201705.327151:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.327153:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.327171:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.327173:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.327183:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450994176, count: 4096 00000080:00200000:1.0:1788201705.327189:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450994176, bytes: 4096 00000080:00200000:1.0:1788201705.327191:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.327193:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.327194:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450994176, 450998272) 0 0 00000080:00200000:1.0:1788201705.327196:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.327199:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110106, 110106] 00020000:00200000:1.0:1788201705.327201:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450994176 chunk: [450994176, 450998272) 450998272, 4096 00020000:00200000:1.0:1788201705.327202:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.327209:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110106, 110106] 00000008:00010000:1.0:1788201705.327212:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.327213:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.327218:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327224:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450994176 450998271) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327230:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327234:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327238:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.327242:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450994176, 450998272) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.327244:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110106 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.327248:0:17463:0:(rw26.c:898:ll_write_end()) pos 450994176, len 4096, copied 4096 00000080:00200000:1.0:1788201705.327250:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000201c9f5d[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110106] 00000080:00200000:1.0:1788201705.327252:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ca08c34e fffffc0001029 2:0 ffff93c0c10f19a0 110106 lru 00000080:00200000:1.0:1788201705.327256:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009a80ff1f 110106: 1< 2 - - > 2< 450994176 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61691 - -> 00000080:00200000:1.0:1788201705.327257:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000201c9f5d 00000080:00200000:1.0:1788201705.327258:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.327259:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.327264:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.327272:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.327274:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.327275:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.327281:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.327282:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.327284:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.327287:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327291:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.327297:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.327298:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.327301:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.327303:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110106, 110106] 00000080:00200000:1.0:1788201705.327309:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.327311:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.327328:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.327331:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.327341:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 450998272, count: 4096 00000080:00200000:1.0:1788201705.327346:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 450998272, bytes: 4096 00000080:00200000:1.0:1788201705.327349:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.327352:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.327353:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [450998272, 451002368) 0 0 00000080:00200000:1.0:1788201705.327356:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.327360:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110107, 110107] 00020000:00200000:1.0:1788201705.327363:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 450998272 chunk: [450998272, 451002368) 451002368, 4096 00020000:00200000:1.0:1788201705.327365:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.327376:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110107, 110107] 00000008:00010000:1.0:1788201705.327382:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.327384:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.327391:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327401:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (450998272 451002367) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327407:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327411:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327416:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.327420:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [450998272, 451002368) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.327424:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110107 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.327428:0:17463:0:(rw26.c:898:ll_write_end()) pos 450998272, len 4096, copied 4096 00000080:00200000:1.0:1788201705.327430:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000bf85851[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110107] 00000080:00200000:1.0:1788201705.327432:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000eeb8c3a9 fffffc0001029 2:0 ffff93c0c10f1b60 110107 lru 00000080:00200000:1.0:1788201705.327436:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000acc17896 110107: 1< 2 - - > 2< 450998272 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61692 - -> 00000080:00200000:1.0:1788201705.327437:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000bf85851 00000080:00200000:1.0:1788201705.327438:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.327439:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.327445:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.327453:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.327455:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.327456:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.327463:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.327465:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.327466:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.327468:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327473:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.327478:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.327480:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.327482:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.327484:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110107, 110107] 00000080:00200000:1.0:1788201705.327491:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.327493:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.327511:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.327514:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.327524:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451002368, count: 4096 00000080:00200000:1.0:1788201705.327530:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451002368, bytes: 4096 00000080:00200000:1.0:1788201705.327532:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.327534:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.327535:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451002368, 451006464) 0 0 00000080:00200000:1.0:1788201705.327536:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.327539:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110108, 110108] 00020000:00200000:1.0:1788201705.327541:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451002368 chunk: [451002368, 451006464) 451006464, 4096 00020000:00200000:1.0:1788201705.327543:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.327549:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110108, 110108] 00000008:00010000:1.0:1788201705.327552:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.327553:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.327558:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327564:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451002368 451006463) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327569:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327574:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327578:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.327583:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451002368, 451006464) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.327586:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110108 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.327589:0:17463:0:(rw26.c:898:ll_write_end()) pos 451002368, len 4096, copied 4096 00000080:00200000:1.0:1788201705.327592:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000827fe977[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110108] 00000080:00200000:1.0:1788201705.327593:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b02436bc fffffc0001029 2:0 ffff93c0c10f1c40 110108 lru 00000080:00200000:1.0:1788201705.327597:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002256a4d7 110108: 1< 2 - - > 2< 451002368 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61693 - -> 00000080:00200000:1.0:1788201705.327598:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000827fe977 00000080:00200000:1.0:1788201705.327599:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.327600:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.327606:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.327613:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.327615:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.327616:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.327622:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.327624:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.327625:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.327627:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327632:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.327636:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.327638:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.327640:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.327642:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110108, 110108] 00000080:00200000:1.0:1788201705.327649:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.327650:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.327668:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.327670:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.327680:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451006464, count: 4096 00000080:00200000:1.0:1788201705.327686:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451006464, bytes: 4096 00000080:00200000:1.0:1788201705.327688:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.327690:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.327691:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451006464, 451010560) 0 0 00000080:00200000:1.0:1788201705.327693:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.327696:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110109, 110109] 00020000:00200000:1.0:1788201705.327697:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451006464 chunk: [451006464, 451010560) 451010560, 4096 00020000:00200000:1.0:1788201705.327699:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.327705:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110109, 110109] 00000008:00010000:1.0:1788201705.327708:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.327709:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.327714:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327720:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451006464 451010559) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327725:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327729:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327733:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.327741:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451006464, 451010560) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.327745:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110109 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.327748:0:17463:0:(rw26.c:898:ll_write_end()) pos 451006464, len 4096, copied 4096 00000080:00200000:1.0:1788201705.327750:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000003b8330b[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110109] 00000080:00200000:1.0:1788201705.327752:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000062434493 fffffc0001029 2:0 ffff93c0c10f1540 110109 lru 00000080:00200000:1.0:1788201705.327756:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000263cd6bd 110109: 1< 2 - - > 2< 451006464 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61694 - -> 00000080:00200000:1.0:1788201705.327757:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000003b8330b 00000080:00200000:1.0:1788201705.327758:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.327759:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.327764:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.327771:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.327773:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.327774:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.327780:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.327782:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.327783:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.327785:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327790:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.327795:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.327796:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.327799:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.327801:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110109, 110109] 00000080:00200000:1.0:1788201705.327807:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.327809:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.327827:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.327840:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.327851:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451010560, count: 4096 00000080:00200000:1.0:1788201705.327857:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451010560, bytes: 4096 00000080:00200000:1.0:1788201705.327859:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.327861:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.327862:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451010560, 451014656) 0 0 00000080:00200000:1.0:1788201705.327864:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.327867:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110110, 110110] 00020000:00200000:1.0:1788201705.327869:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451010560 chunk: [451010560, 451014656) 451014656, 4096 00020000:00200000:1.0:1788201705.327870:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.327877:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110110, 110110] 00000008:00010000:1.0:1788201705.327880:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.327882:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.327887:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327892:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451010560 451014655) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327898:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327902:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327909:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.327914:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451010560, 451014656) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.327917:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110110 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.327920:0:17463:0:(rw26.c:898:ll_write_end()) pos 451010560, len 4096, copied 4096 00000080:00200000:1.0:1788201705.327922:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000cd9f981a[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110110] 00000080:00200000:1.0:1788201705.327924:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000025208fec fffffc0001029 2:0 ffff93c0c10f1460 110110 lru 00000080:00200000:1.0:1788201705.327927:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f77f131d 110110: 1< 2 - - > 2< 451010560 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61695 - -> 00000080:00200000:1.0:1788201705.327928:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000cd9f981a 00000080:00200000:1.0:1788201705.327929:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.327930:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.327936:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.327943:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.327945:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.327946:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.327952:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.327954:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.327955:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.327957:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.327962:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.327970:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.327971:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.327974:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.327976:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110110, 110110] 00000080:00200000:1.0:1788201705.327982:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.327984:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.328002:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.328004:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.328015:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451014656, count: 4096 00000080:00200000:1.0:1788201705.328024:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451014656, bytes: 4096 00000080:00200000:1.0:1788201705.328027:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.328031:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.328032:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451014656, 451018752) 0 0 00000080:00200000:1.0:1788201705.328035:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.328050:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110111, 110111] 00020000:00200000:1.0:1788201705.328053:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451014656 chunk: [451014656, 451018752) 451018752, 4096 00020000:00200000:1.0:1788201705.328056:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.328068:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110111, 110111] 00000008:00010000:1.0:1788201705.328074:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.328075:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.328081:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328087:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451014656 451018751) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328093:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328101:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328105:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.328110:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451014656, 451018752) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.328113:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110111 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.328116:0:17463:0:(rw26.c:898:ll_write_end()) pos 451014656, len 4096, copied 4096 00000080:00200000:1.0:1788201705.328118:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000689a4a1b[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110111] 00000080:00200000:1.0:1788201705.328120:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002c885830 fffffc0001029 2:0 ffff93c0c10f10e0 110111 lru 00000080:00200000:1.0:1788201705.328124:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c830bafb 110111: 1< 2 - - > 2< 451014656 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61696 - -> 00000080:00200000:1.0:1788201705.328125:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000689a4a1b 00000080:00200000:1.0:1788201705.328125:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.328126:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.328132:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.328140:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.328142:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.328143:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.328149:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.328151:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.328152:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.328155:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328163:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.328168:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.328169:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.328171:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.328173:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110111, 110111] 00000080:00200000:1.0:1788201705.328180:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.328182:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.328200:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.328202:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.328213:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451018752, count: 4096 00000080:00200000:1.0:1788201705.328218:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451018752, bytes: 4096 00000080:00200000:1.0:1788201705.328220:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.328222:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.328223:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451018752, 451022848) 0 0 00000080:00200000:1.0:1788201705.328225:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.328228:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110112, 110112] 00020000:00200000:1.0:1788201705.328230:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451018752 chunk: [451018752, 451022848) 451022848, 4096 00020000:00200000:1.0:1788201705.328231:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.328238:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110112, 110112] 00000008:00010000:1.0:1788201705.328241:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.328242:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.328247:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328253:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451018752 451022847) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328262:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328266:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328270:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.328274:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451018752, 451022848) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.328277:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110112 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.328281:0:17463:0:(rw26.c:898:ll_write_end()) pos 451018752, len 4096, copied 4096 00000080:00200000:1.0:1788201705.328283:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c940cde9[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110112] 00000080:00200000:1.0:1788201705.328285:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e6a99dc7 fffffc0001029 2:0 ffff93c0c10f1700 110112 lru 00000080:00200000:1.0:1788201705.328289:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b8f1f63a 110112: 1< 2 - - > 2< 451018752 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61697 - -> 00000080:00200000:1.0:1788201705.328290:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c940cde9 00000080:00200000:1.0:1788201705.328291:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.328292:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.328297:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.328305:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.328306:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.328308:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.328314:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.328315:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.328317:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.328319:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328324:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.328329:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.328330:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.328333:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.328335:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110112, 110112] 00000080:00200000:1.0:1788201705.328341:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.328343:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.328368:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.328372:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.328388:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451022848, count: 4096 00000080:00200000:1.0:1788201705.328399:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451022848, bytes: 4096 00000080:00200000:1.0:1788201705.328401:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.328403:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.328404:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451022848, 451026944) 0 0 00000080:00200000:1.0:1788201705.328406:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.328409:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110113, 110113] 00020000:00200000:1.0:1788201705.328411:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451022848 chunk: [451022848, 451026944) 451026944, 4096 00020000:00200000:1.0:1788201705.328412:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.328420:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110113, 110113] 00000008:00010000:1.0:1788201705.328423:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.328424:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.328430:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328439:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451022848 451026943) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328444:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328449:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328453:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.328457:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451022848, 451026944) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.328460:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110113 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.328464:0:17463:0:(rw26.c:898:ll_write_end()) pos 451022848, len 4096, copied 4096 00000080:00200000:1.0:1788201705.328466:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000085d0051c[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110113] 00000080:00200000:1.0:1788201705.328468:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000097d2272c fffffc0001029 2:0 ffff93c0c10f1000 110113 lru 00000080:00200000:1.0:1788201705.328472:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000063f74bf2 110113: 1< 2 - - > 2< 451022848 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61698 - -> 00000080:00200000:1.0:1788201705.328473:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000085d0051c 00000080:00200000:1.0:1788201705.328474:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.328477:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.328483:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.328491:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.328493:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.328494:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.328501:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.328503:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.328504:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.328506:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328512:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.328516:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.328518:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.328520:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.328522:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110113, 110113] 00000080:00200000:1.0:1788201705.328528:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.328530:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.328548:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.328550:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.328560:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451026944, count: 4096 00000080:00200000:1.0:1788201705.328566:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451026944, bytes: 4096 00000080:00200000:1.0:1788201705.328568:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.328570:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.328571:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451026944, 451031040) 0 0 00000080:00200000:1.0:1788201705.328572:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.328576:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110114, 110114] 00020000:00200000:1.0:1788201705.328577:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451026944 chunk: [451026944, 451031040) 451031040, 4096 00020000:00200000:1.0:1788201705.328579:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.328585:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110114, 110114] 00000008:00010000:1.0:1788201705.328588:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.328590:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.328595:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328600:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451026944 451031039) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328606:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328610:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328614:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.328618:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451026944, 451031040) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.328620:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110114 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.328624:0:17463:0:(rw26.c:898:ll_write_end()) pos 451026944, len 4096, copied 4096 00000080:00200000:1.0:1788201705.328626:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000055d8dc3e[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110114] 00000080:00200000:1.0:1788201705.328628:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000081a518d3 fffffc0001029 2:0 ffff93c0c10f1a80 110114 lru 00000080:00200000:1.0:1788201705.328631:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e1d8767e 110114: 1< 2 - - > 2< 451026944 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61699 - -> 00000080:00200000:1.0:1788201705.328633:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000055d8dc3e 00000080:00200000:1.0:1788201705.328634:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.328635:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.328640:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.328647:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.328649:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.328650:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.328656:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.328657:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.328659:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.328661:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328666:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.328670:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.328672:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.328674:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.328676:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110114, 110114] 00000080:00200000:1.0:1788201705.328683:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.328685:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.328702:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.328704:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.328714:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451031040, count: 4096 00000080:00200000:1.0:1788201705.328720:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451031040, bytes: 4096 00000080:00200000:1.0:1788201705.328721:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.328723:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.328725:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451031040, 451035136) 0 0 00000080:00200000:1.0:1788201705.328726:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.328730:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110115, 110115] 00020000:00200000:1.0:1788201705.328731:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451031040 chunk: [451031040, 451035136) 451035136, 4096 00020000:00200000:1.0:1788201705.328733:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.328739:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110115, 110115] 00000008:00010000:1.0:1788201705.328743:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.328744:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.328749:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328754:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451031040 451035135) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328760:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328764:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328768:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.328773:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451031040, 451035136) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.328776:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110115 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.328779:0:17463:0:(rw26.c:898:ll_write_end()) pos 451031040, len 4096, copied 4096 00000080:00200000:1.0:1788201705.328782:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000098232b0f[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110115] 00000080:00200000:1.0:1788201705.328784:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000eb6c68b0 fffffc0001029 2:0 ffff93c0c10f1380 110115 lru 00000080:00200000:1.0:1788201705.328787:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ffae9b21 110115: 1< 2 - - > 2< 451031040 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61700 - -> 00000080:00200000:1.0:1788201705.328789:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000098232b0f 00000080:00200000:1.0:1788201705.328789:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.328791:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.328796:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.328807:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.328808:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.328810:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.328815:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.328817:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.328819:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.328821:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328825:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.328846:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.328848:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.328851:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.328853:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110115, 110115] 00000080:00200000:1.0:1788201705.328860:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.328862:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.328880:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.328882:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.328894:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451035136, count: 4096 00000080:00200000:1.0:1788201705.328900:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451035136, bytes: 4096 00000080:00200000:1.0:1788201705.328902:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.328904:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.328905:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451035136, 451039232) 0 0 00000080:00200000:1.0:1788201705.328907:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.328910:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110116, 110116] 00020000:00200000:1.0:1788201705.328912:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451035136 chunk: [451035136, 451039232) 451039232, 4096 00020000:00200000:1.0:1788201705.328914:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.328920:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110116, 110116] 00000008:00010000:1.0:1788201705.328923:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.328924:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.328929:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328935:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451035136 451039231) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328940:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328945:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.328949:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.328953:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451035136, 451039232) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.328956:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110116 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.328960:0:17463:0:(rw26.c:898:ll_write_end()) pos 451035136, len 4096, copied 4096 00000080:00200000:1.0:1788201705.328962:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b1ad4d1b[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110116] 00000080:00200000:1.0:1788201705.328964:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005298820e fffffc0001029 2:0 ffff93c0c10f18c0 110116 lru 00000080:00200000:1.0:1788201705.328967:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e14549bb 110116: 1< 2 - - > 2< 451035136 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61701 - -> 00000080:00200000:1.0:1788201705.328969:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b1ad4d1b 00000080:00200000:1.0:1788201705.328969:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.328971:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.328977:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.328984:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.328986:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.328987:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.328993:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.328994:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.328996:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.328998:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329003:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.329007:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.329009:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.329011:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.329014:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110116, 110116] 00000080:00200000:1.0:1788201705.329024:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.329027:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.329065:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.329069:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.329083:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451039232, count: 4096 00000080:00200000:1.0:1788201705.329093:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451039232, bytes: 4096 00000080:00200000:1.0:1788201705.329095:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.329097:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.329098:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451039232, 451043328) 0 0 00000080:00200000:1.0:1788201705.329100:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.329103:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110117, 110117] 00020000:00200000:1.0:1788201705.329105:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451039232 chunk: [451039232, 451043328) 451043328, 4096 00020000:00200000:1.0:1788201705.329107:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.329114:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110117, 110117] 00000008:00010000:1.0:1788201705.329117:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.329119:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.329124:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329130:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451039232 451043327) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329136:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329140:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329144:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.329149:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451039232, 451043328) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.329152:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110117 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.329156:0:17463:0:(rw26.c:898:ll_write_end()) pos 451039232, len 4096, copied 4096 00000080:00200000:1.0:1788201705.329158:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000dd3e3988[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110117] 00000080:00200000:1.0:1788201705.329160:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002d5ff549 fffffc0001029 2:0 ffff93c0c10f1ee0 110117 lru 00000080:00200000:1.0:1788201705.329164:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000ae83198 110117: 1< 2 - - > 2< 451039232 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61702 - -> 00000080:00200000:1.0:1788201705.329166:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000dd3e3988 00000080:00200000:1.0:1788201705.329167:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.329168:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.329174:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.329181:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.329183:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.329184:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.329191:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.329192:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.329194:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.329196:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329200:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.329205:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.329206:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.329209:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.329211:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110117, 110117] 00000080:00200000:1.0:1788201705.329217:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.329219:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.329237:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.329239:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.329250:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451043328, count: 4096 00000080:00200000:1.0:1788201705.329255:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451043328, bytes: 4096 00000080:00200000:1.0:1788201705.329257:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.329259:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.329260:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451043328, 451047424) 0 0 00000080:00200000:1.0:1788201705.329262:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.329265:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110118, 110118] 00020000:00200000:1.0:1788201705.329267:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451043328 chunk: [451043328, 451047424) 451047424, 4096 00020000:00200000:1.0:1788201705.329268:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.329275:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110118, 110118] 00000008:00010000:1.0:1788201705.329278:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.329279:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.329284:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329290:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451043328 451047423) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329295:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329299:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329303:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.329308:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451043328, 451047424) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.329311:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110118 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.329314:0:17463:0:(rw26.c:898:ll_write_end()) pos 451043328, len 4096, copied 4096 00000080:00200000:1.0:1788201705.329316:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000bee4ebf[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110118] 00000080:00200000:1.0:1788201705.329318:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007e13503e fffffc0001029 2:0 ffff93c0c10f17e0 110118 lru 00000080:00200000:1.0:1788201705.329322:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000cc258c1 110118: 1< 2 - - > 2< 451043328 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61703 - -> 00000080:00200000:1.0:1788201705.329324:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000bee4ebf 00000080:00200000:1.0:1788201705.329324:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.329326:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.329331:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.329338:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.329340:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.329341:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.329348:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.329351:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.329353:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.329356:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329364:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.329372:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.329374:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.329379:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.329382:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110118, 110118] 00000080:00200000:1.0:1788201705.329392:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.329395:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.329415:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.329417:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.329428:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451047424, count: 4096 00000080:00200000:1.0:1788201705.329434:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451047424, bytes: 4096 00000080:00200000:1.0:1788201705.329436:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.329438:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.329439:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451047424, 451051520) 0 0 00000080:00200000:1.0:1788201705.329440:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.329444:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110119, 110119] 00020000:00200000:1.0:1788201705.329446:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451047424 chunk: [451047424, 451051520) 451051520, 4096 00020000:00200000:1.0:1788201705.329447:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.329454:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110119, 110119] 00000008:00010000:1.0:1788201705.329457:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.329458:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.329464:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329470:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451047424 451051519) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329475:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329480:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329484:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.329488:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451047424, 451051520) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.329491:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110119 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.329495:0:17463:0:(rw26.c:898:ll_write_end()) pos 451047424, len 4096, copied 4096 00000080:00200000:1.0:1788201705.329498:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b2782d06[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110119] 00000080:00200000:1.0:1788201705.329499:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000036caf17a fffffc0001029 2:0 ffff93c0c10f12a0 110119 lru 00000080:00200000:1.0:1788201705.329503:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000e5b7e5a 110119: 1< 2 - - > 2< 451047424 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61704 - -> 00000080:00200000:1.0:1788201705.329504:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b2782d06 00000080:00200000:1.0:1788201705.329505:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.329506:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.329512:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.329520:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.329522:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.329523:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.329529:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.329531:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.329533:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.329535:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329540:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.329545:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.329546:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.329549:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.329550:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110119, 110119] 00000080:00200000:1.0:1788201705.329557:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.329559:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.329576:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.329579:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.329588:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451051520, count: 4096 00000080:00200000:1.0:1788201705.329594:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451051520, bytes: 4096 00000080:00200000:1.0:1788201705.329596:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.329598:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.329599:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451051520, 451055616) 0 0 00000080:00200000:1.0:1788201705.329600:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.329603:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110120, 110120] 00020000:00200000:1.0:1788201705.329605:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451051520 chunk: [451051520, 451055616) 451055616, 4096 00020000:00200000:1.0:1788201705.329607:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.329613:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110120, 110120] 00000008:00010000:1.0:1788201705.329616:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.329617:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.329622:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329628:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451051520 451055615) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329633:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329637:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329641:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.329646:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451051520, 451055616) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.329649:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110120 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.329652:0:17463:0:(rw26.c:898:ll_write_end()) pos 451051520, len 4096, copied 4096 00000080:00200000:1.0:1788201705.329655:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006642b4e6[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110120] 00000080:00200000:1.0:1788201705.329656:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d44366a1 fffffc0001029 2:0 ffff93c0c10f27e0 110120 lru 00000080:00200000:1.0:1788201705.329660:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000dd392c46 110120: 1< 2 - - > 2< 451051520 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61705 - -> 00000080:00200000:1.0:1788201705.329661:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006642b4e6 00000080:00200000:1.0:1788201705.329662:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.329663:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.329669:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.329676:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.329678:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.329679:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.329685:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.329686:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.329688:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.329690:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329695:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.329700:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.329701:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.329703:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.329705:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110120, 110120] 00000080:00200000:1.0:1788201705.329712:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.329713:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.329731:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.329733:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.329743:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451055616, count: 4096 00000080:00200000:1.0:1788201705.329749:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451055616, bytes: 4096 00000080:00200000:1.0:1788201705.329750:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.329753:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.329754:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451055616, 451059712) 0 0 00000080:00200000:1.0:1788201705.329755:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.329758:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110121, 110121] 00020000:00200000:1.0:1788201705.329760:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451055616 chunk: [451055616, 451059712) 451059712, 4096 00020000:00200000:1.0:1788201705.329762:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.329768:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110121, 110121] 00000008:00010000:1.0:1788201705.329771:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.329772:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.329778:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329783:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451055616 451059711) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329789:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329793:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329797:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.329805:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451055616, 451059712) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.329808:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110121 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.329811:0:17463:0:(rw26.c:898:ll_write_end()) pos 451055616, len 4096, copied 4096 00000080:00200000:1.0:1788201705.329813:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001b4c499d[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110121] 00000080:00200000:1.0:1788201705.329815:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000023e4f867 fffffc0001029 2:0 ffff93c0c10f22a0 110121 lru 00000080:00200000:1.0:1788201705.329819:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000329cbed3 110121: 1< 2 - - > 2< 451055616 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61706 - -> 00000080:00200000:1.0:1788201705.329820:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001b4c499d 00000080:00200000:1.0:1788201705.329821:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.329822:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.329827:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.329848:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.329850:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.329851:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.329857:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.329859:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.329860:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.329863:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329868:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.329873:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.329874:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.329877:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.329879:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110121, 110121] 00000080:00200000:1.0:1788201705.329887:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.329890:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.329916:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.329920:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.329934:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451059712, count: 4096 00000080:00200000:1.0:1788201705.329944:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451059712, bytes: 4096 00000080:00200000:1.0:1788201705.329947:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.329950:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.329952:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451059712, 451063808) 0 0 00000080:00200000:1.0:1788201705.329954:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.329958:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110122, 110122] 00020000:00200000:1.0:1788201705.329961:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451059712 chunk: [451059712, 451063808) 451063808, 4096 00020000:00200000:1.0:1788201705.329963:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.329975:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110122, 110122] 00000008:00010000:1.0:1788201705.329980:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.329982:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.329990:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.329999:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451059712 451063807) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330008:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330014:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330026:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.330035:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451059712, 451063808) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.330051:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110122 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.330056:0:17463:0:(rw26.c:898:ll_write_end()) pos 451059712, len 4096, copied 4096 00000080:00200000:1.0:1788201705.330061:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a48c7a7d[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110122] 00000080:00200000:1.0:1788201705.330064:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000379b256e fffffc0001029 2:0 ffff93c0c10f2e00 110122 lru 00000080:00200000:1.0:1788201705.330069:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006255244b 110122: 1< 2 - - > 2< 451059712 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61707 - -> 00000080:00200000:1.0:1788201705.330071:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a48c7a7d 00000080:00200000:1.0:1788201705.330072:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.330073:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.330080:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.330088:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.330090:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.330092:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.330098:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.330100:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.330102:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.330110:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330115:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.330124:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.330126:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.330129:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.330131:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110122, 110122] 00000080:00200000:1.0:1788201705.330138:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.330140:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.330159:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.330161:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.330172:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451063808, count: 4096 00000080:00200000:1.0:1788201705.330178:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451063808, bytes: 4096 00000080:00200000:1.0:1788201705.330180:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.330182:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.330183:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451063808, 451067904) 0 0 00000080:00200000:1.0:1788201705.330184:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.330188:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110123, 110123] 00020000:00200000:1.0:1788201705.330190:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451063808 chunk: [451063808, 451067904) 451067904, 4096 00020000:00200000:1.0:1788201705.330191:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.330198:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110123, 110123] 00000008:00010000:1.0:1788201705.330201:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.330203:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.330208:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330214:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451063808 451067903) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330220:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330228:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330232:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.330236:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451063808, 451067904) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.330239:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110123 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.330243:0:17463:0:(rw26.c:898:ll_write_end()) pos 451063808, len 4096, copied 4096 00000080:00200000:1.0:1788201705.330245:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000039872a24[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110123] 00000080:00200000:1.0:1788201705.330246:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004b72fa5c fffffc0001029 2:0 ffff93c0c10f21c0 110123 lru 00000080:00200000:1.0:1788201705.330250:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000085bc4c2a 110123: 1< 2 - - > 2< 451063808 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61708 - -> 00000080:00200000:1.0:1788201705.330251:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000039872a24 00000080:00200000:1.0:1788201705.330252:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.330253:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.330259:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.330266:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.330268:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.330269:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.330275:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.330277:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.330278:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.330281:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330289:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.330293:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.330295:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.330297:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.330299:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110123, 110123] 00000080:00200000:1.0:1788201705.330305:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.330307:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.330325:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.330327:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.330337:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451067904, count: 4096 00000080:00200000:1.0:1788201705.330343:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451067904, bytes: 4096 00000080:00200000:1.0:1788201705.330345:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.330348:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.330349:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451067904, 451072000) 0 0 00000080:00200000:1.0:1788201705.330352:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.330356:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110124, 110124] 00020000:00200000:1.0:1788201705.330359:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451067904 chunk: [451067904, 451072000) 451072000, 4096 00020000:00200000:1.0:1788201705.330361:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.330372:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110124, 110124] 00000008:00010000:1.0:1788201705.330377:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.330380:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.330388:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330397:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451067904 451071999) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330407:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330411:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330415:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.330420:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451067904, 451072000) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.330423:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110124 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.330427:0:17463:0:(rw26.c:898:ll_write_end()) pos 451067904, len 4096, copied 4096 00000080:00200000:1.0:1788201705.330429:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000644b6dce[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110124] 00000080:00200000:1.0:1788201705.330431:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001c95898a fffffc0001029 2:0 ffff93c0c10f2620 110124 lru 00000080:00200000:1.0:1788201705.330435:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003f6ca282 110124: 1< 2 - - > 2< 451067904 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61709 - -> 00000080:00200000:1.0:1788201705.330436:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000644b6dce 00000080:00200000:1.0:1788201705.330437:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.330438:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.330444:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.330452:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.330454:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.330455:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.330461:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.330463:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.330465:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.330467:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330472:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.330477:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.330479:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.330481:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.330483:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110124, 110124] 00000080:00200000:1.0:1788201705.330490:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.330492:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.330510:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.330512:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.330522:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451072000, count: 4096 00000080:00200000:1.0:1788201705.330528:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451072000, bytes: 4096 00000080:00200000:1.0:1788201705.330530:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.330532:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.330533:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451072000, 451076096) 0 0 00000080:00200000:1.0:1788201705.330534:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.330538:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110125, 110125] 00020000:00200000:1.0:1788201705.330539:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451072000 chunk: [451072000, 451076096) 451076096, 4096 00020000:00200000:1.0:1788201705.330541:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.330547:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110125, 110125] 00000008:00010000:1.0:1788201705.330550:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.330552:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.330557:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330566:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451072000 451076095) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330571:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330575:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330579:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.330584:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451072000, 451076096) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.330587:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110125 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.330590:0:17463:0:(rw26.c:898:ll_write_end()) pos 451072000, len 4096, copied 4096 00000080:00200000:1.0:1788201705.330592:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a369c996[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110125] 00000080:00200000:1.0:1788201705.330595:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000450a6bb4 fffffc0001029 2:0 ffff93c0c10f2d20 110125 lru 00000080:00200000:1.0:1788201705.330598:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000014de9c41 110125: 1< 2 - - > 2< 451072000 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61710 - -> 00000080:00200000:1.0:1788201705.330599:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a369c996 00000080:00200000:1.0:1788201705.330600:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.330601:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.330607:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.330614:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.330616:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.330617:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.330623:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.330625:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.330626:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.330629:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330634:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.330638:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.330640:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.330642:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.330644:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110125, 110125] 00000080:00200000:1.0:1788201705.330650:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.330652:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.330669:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.330672:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.330681:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451076096, count: 4096 00000080:00200000:1.0:1788201705.330687:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451076096, bytes: 4096 00000080:00200000:1.0:1788201705.330689:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.330691:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.330692:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451076096, 451080192) 0 0 00000080:00200000:1.0:1788201705.330693:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.330697:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110126, 110126] 00020000:00200000:1.0:1788201705.330698:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451076096 chunk: [451076096, 451080192) 451080192, 4096 00020000:00200000:1.0:1788201705.330700:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.330706:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110126, 110126] 00000008:00010000:1.0:1788201705.330709:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.330710:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.330715:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330721:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451076096 451080191) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330727:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330731:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330735:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.330739:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451076096, 451080192) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.330741:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110126 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.330745:0:17463:0:(rw26.c:898:ll_write_end()) pos 451076096, len 4096, copied 4096 00000080:00200000:1.0:1788201705.330747:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000076c78be8[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110126] 00000080:00200000:1.0:1788201705.330748:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000097b7f08b fffffc0001029 2:0 ffff93c0c10f29a0 110126 lru 00000080:00200000:1.0:1788201705.330752:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c81fc589 110126: 1< 2 - - > 2< 451076096 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61711 - -> 00000080:00200000:1.0:1788201705.330753:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000076c78be8 00000080:00200000:1.0:1788201705.330754:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.330755:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.330761:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.330768:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.330770:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.330771:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.330777:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.330779:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.330780:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.330782:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330787:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.330791:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.330793:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.330795:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.330797:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110126, 110126] 00000080:00200000:1.0:1788201705.330804:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.330806:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.330823:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.330825:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.330847:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451080192, count: 4096 00000080:00200000:1.0:1788201705.330854:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451080192, bytes: 4096 00000080:00200000:1.0:1788201705.330856:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.330858:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.330859:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451080192, 451084288) 0 0 00000080:00200000:1.0:1788201705.330860:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.330864:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110127, 110127] 00020000:00200000:1.0:1788201705.330865:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451080192 chunk: [451080192, 451084288) 451084288, 4096 00020000:00200000:1.0:1788201705.330867:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.330873:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110127, 110127] 00000008:00010000:1.0:1788201705.330876:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.330878:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.330883:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330889:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451080192 451084287) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330894:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330898:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330902:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.330907:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451080192, 451084288) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.330910:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110127 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.330913:0:17463:0:(rw26.c:898:ll_write_end()) pos 451080192, len 4096, copied 4096 00000080:00200000:1.0:1788201705.330915:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001e85af02[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110127] 00000080:00200000:1.0:1788201705.330917:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f41f0b50 fffffc0001029 2:0 ffff93c0c10f2b60 110127 lru 00000080:00200000:1.0:1788201705.330920:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c2fbd41a 110127: 1< 2 - - > 2< 451080192 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61712 - -> 00000080:00200000:1.0:1788201705.330922:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001e85af02 00000080:00200000:1.0:1788201705.330923:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.330924:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.330929:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.330940:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.330942:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.330943:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.330949:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.330951:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.330952:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.330955:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.330959:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.330964:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.330965:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.330967:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.330969:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110127, 110127] 00000080:00200000:1.0:1788201705.330976:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.330978:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.330995:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.330998:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.331007:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451084288, count: 4096 00000080:00200000:1.0:1788201705.331015:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451084288, bytes: 4096 00000080:00200000:1.0:1788201705.331018:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.331021:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.331022:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451084288, 451088384) 0 0 00000080:00200000:1.0:1788201705.331025:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.331029:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110128, 110128] 00020000:00200000:1.0:1788201705.331033:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451084288 chunk: [451084288, 451088384) 451088384, 4096 00020000:00200000:1.0:1788201705.331035:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.331057:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110128, 110128] 00000008:00010000:1.0:1788201705.331062:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.331064:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.331072:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331078:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451084288 451088383) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331084:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331088:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331092:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.331097:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451084288, 451088384) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.331101:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110128 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.331105:0:17463:0:(rw26.c:898:ll_write_end()) pos 451084288, len 4096, copied 4096 00000080:00200000:1.0:1788201705.331107:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f857f471[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110128] 00000080:00200000:1.0:1788201705.331109:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000390ad09e fffffc0001029 2:0 ffff93c0c10f2c40 110128 lru 00000080:00200000:1.0:1788201705.331113:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000500bbf34 110128: 1< 2 - - > 2< 451084288 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61713 - -> 00000080:00200000:1.0:1788201705.331114:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f857f471 00000080:00200000:1.0:1788201705.331115:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.331117:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.331122:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.331130:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.331132:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.331133:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.331139:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.331141:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.331143:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.331145:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331150:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.331155:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.331156:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.331158:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.331160:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110128, 110128] 00000080:00200000:1.0:1788201705.331167:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.331169:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.331187:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.331189:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.331199:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451088384, count: 4096 00000080:00200000:1.0:1788201705.331205:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451088384, bytes: 4096 00000080:00200000:1.0:1788201705.331207:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.331209:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.331210:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451088384, 451092480) 0 0 00000080:00200000:1.0:1788201705.331212:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.331215:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110129, 110129] 00020000:00200000:1.0:1788201705.331217:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451088384 chunk: [451088384, 451092480) 451092480, 4096 00020000:00200000:1.0:1788201705.331218:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.331225:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110129, 110129] 00000008:00010000:1.0:1788201705.331228:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.331229:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.331234:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331240:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451088384 451092479) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331245:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331249:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331253:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.331258:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451088384, 451092480) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.331261:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110129 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.331264:0:17463:0:(rw26.c:898:ll_write_end()) pos 451088384, len 4096, copied 4096 00000080:00200000:1.0:1788201705.331266:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b97ed336[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110129] 00000080:00200000:1.0:1788201705.331268:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000026d83372 fffffc0001029 2:0 ffff93c0c10f2540 110129 lru 00000080:00200000:1.0:1788201705.331272:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009757cd95 110129: 1< 2 - - > 2< 451088384 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61714 - -> 00000080:00200000:1.0:1788201705.331274:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b97ed336 00000080:00200000:1.0:1788201705.331275:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.331276:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.331282:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.331289:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.331291:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.331292:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.331298:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.331299:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.331301:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.331303:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331308:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.331312:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.331313:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.331316:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.331318:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110129, 110129] 00000080:00200000:1.0:1788201705.331324:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.331326:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.331344:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.331346:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.331360:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451092480, count: 4096 00000080:00200000:1.0:1788201705.331371:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451092480, bytes: 4096 00000080:00200000:1.0:1788201705.331374:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.331377:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.331379:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451092480, 451096576) 0 0 00000080:00200000:1.0:1788201705.331381:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.331386:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110130, 110130] 00020000:00200000:1.0:1788201705.331389:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451092480 chunk: [451092480, 451096576) 451096576, 4096 00020000:00200000:1.0:1788201705.331392:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.331402:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110130, 110130] 00000008:00010000:1.0:1788201705.331405:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.331407:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.331412:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331418:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451092480 451096575) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331424:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331428:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331432:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.331437:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451092480, 451096576) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.331440:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110130 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.331444:0:17463:0:(rw26.c:898:ll_write_end()) pos 451092480, len 4096, copied 4096 00000080:00200000:1.0:1788201705.331446:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000dc14f70c[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110130] 00000080:00200000:1.0:1788201705.331448:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008216ee28 fffffc0001029 2:0 ffff93c0c10f2460 110130 lru 00000080:00200000:1.0:1788201705.331452:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ebdb6bd5 110130: 1< 2 - - > 2< 451092480 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61715 - -> 00000080:00200000:1.0:1788201705.331453:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000dc14f70c 00000080:00200000:1.0:1788201705.331454:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.331455:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.331461:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.331468:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.331470:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.331471:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.331477:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.331479:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.331480:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.331482:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331487:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.331493:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.331494:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.331496:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.331498:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110130, 110130] 00000080:00200000:1.0:1788201705.331505:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.331507:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.331524:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.331527:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.331537:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451096576, count: 4096 00000080:00200000:1.0:1788201705.331543:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451096576, bytes: 4096 00000080:00200000:1.0:1788201705.331545:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.331547:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.331548:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451096576, 451100672) 0 0 00000080:00200000:1.0:1788201705.331550:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.331553:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110131, 110131] 00020000:00200000:1.0:1788201705.331554:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451096576 chunk: [451096576, 451100672) 451100672, 4096 00020000:00200000:1.0:1788201705.331556:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.331562:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110131, 110131] 00000008:00010000:1.0:1788201705.331566:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.331567:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.331572:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331577:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451096576 451100671) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331583:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331587:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331591:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.331595:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451096576, 451100672) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.331598:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110131 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.331601:0:17463:0:(rw26.c:898:ll_write_end()) pos 451096576, len 4096, copied 4096 00000080:00200000:1.0:1788201705.331604:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002b9e1721[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110131] 00000080:00200000:1.0:1788201705.331606:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007ee94c55 fffffc0001029 2:0 ffff93c0c10f20e0 110131 lru 00000080:00200000:1.0:1788201705.331609:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f8781281 110131: 1< 2 - - > 2< 451096576 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61716 - -> 00000080:00200000:1.0:1788201705.331611:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002b9e1721 00000080:00200000:1.0:1788201705.331612:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.331613:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.331618:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.331625:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.331627:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.331628:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.331634:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.331636:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.331637:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.331640:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331644:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.331649:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.331650:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.331652:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.331654:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110131, 110131] 00000080:00200000:1.0:1788201705.331661:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.331663:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.331680:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.331683:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.331692:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451100672, count: 4096 00000080:00200000:1.0:1788201705.331698:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451100672, bytes: 4096 00000080:00200000:1.0:1788201705.331700:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.331702:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.331703:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451100672, 451104768) 0 0 00000080:00200000:1.0:1788201705.331704:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.331708:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110132, 110132] 00020000:00200000:1.0:1788201705.331709:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451100672 chunk: [451100672, 451104768) 451104768, 4096 00020000:00200000:1.0:1788201705.331711:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.331717:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110132, 110132] 00000008:00010000:1.0:1788201705.331720:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.331721:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.331726:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331732:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451100672 451104767) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331737:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331742:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331746:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.331750:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451100672, 451104768) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.331753:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110132 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.331757:0:17463:0:(rw26.c:898:ll_write_end()) pos 451100672, len 4096, copied 4096 00000080:00200000:1.0:1788201705.331759:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d3ecc84c[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110132] 00000080:00200000:1.0:1788201705.331761:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000098af0e fffffc0001029 2:0 ffff93c0c10f2700 110132 lru 00000080:00200000:1.0:1788201705.331764:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cdd681c6 110132: 1< 2 - - > 2< 451100672 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61717 - -> 00000080:00200000:1.0:1788201705.331766:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d3ecc84c 00000080:00200000:1.0:1788201705.331766:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.331768:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.331773:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.331781:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.331783:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.331785:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.331793:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.331796:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.331798:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.331802:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331811:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.331818:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.331820:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.331823:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.331826:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110132, 110132] 00000080:00200000:1.0:1788201705.331851:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.331855:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.331883:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.331887:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.331900:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451104768, count: 4096 00000080:00200000:1.0:1788201705.331909:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451104768, bytes: 4096 00000080:00200000:1.0:1788201705.331912:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.331915:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.331916:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451104768, 451108864) 0 0 00000080:00200000:1.0:1788201705.331918:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.331923:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110133, 110133] 00020000:00200000:1.0:1788201705.331926:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451104768 chunk: [451104768, 451108864) 451108864, 4096 00020000:00200000:1.0:1788201705.331928:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.331938:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110133, 110133] 00000008:00010000:1.0:1788201705.331943:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.331945:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.331952:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331961:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451104768 451108863) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331969:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331977:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.331983:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.331993:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451104768, 451108864) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.331997:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110133 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.332001:0:17463:0:(rw26.c:898:ll_write_end()) pos 451104768, len 4096, copied 4096 00000080:00200000:1.0:1788201705.332003:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005128f568[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110133] 00000080:00200000:1.0:1788201705.332005:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000023878da fffffc0001029 2:0 ffff93c0c10f2000 110133 lru 00000080:00200000:1.0:1788201705.332009:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008e04ac6e 110133: 1< 2 - - > 2< 451104768 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61718 - -> 00000080:00200000:1.0:1788201705.332011:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005128f568 00000080:00200000:1.0:1788201705.332012:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.332013:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.332022:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.332033:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.332036:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.332049:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.332059:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.332062:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.332065:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.332069:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332076:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.332082:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.332083:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.332087:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.332089:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110133, 110133] 00000080:00200000:1.0:1788201705.332096:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.332098:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.332117:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.332120:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.332130:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451108864, count: 4096 00000080:00200000:1.0:1788201705.332136:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451108864, bytes: 4096 00000080:00200000:1.0:1788201705.332138:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.332140:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.332141:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451108864, 451112960) 0 0 00000080:00200000:1.0:1788201705.332143:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.332146:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110134, 110134] 00020000:00200000:1.0:1788201705.332148:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451108864 chunk: [451108864, 451112960) 451112960, 4096 00020000:00200000:1.0:1788201705.332149:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.332156:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110134, 110134] 00000008:00010000:1.0:1788201705.332159:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.332161:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.332166:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332172:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451108864 451112959) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332177:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332181:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332189:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.332193:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451108864, 451112960) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.332196:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110134 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.332199:0:17463:0:(rw26.c:898:ll_write_end()) pos 451108864, len 4096, copied 4096 00000080:00200000:1.0:1788201705.332202:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002fa1341b[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110134] 00000080:00200000:1.0:1788201705.332204:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000018dfd6a fffffc0001029 2:0 ffff93c0c10f2a80 110134 lru 00000080:00200000:1.0:1788201705.332207:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001a48a7b5 110134: 1< 2 - - > 2< 451108864 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61719 - -> 00000080:00200000:1.0:1788201705.332209:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002fa1341b 00000080:00200000:1.0:1788201705.332209:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.332211:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.332216:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.332224:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.332226:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.332227:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.332234:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.332235:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.332237:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.332239:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332244:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.332253:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.332254:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.332257:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.332258:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110134, 110134] 00000080:00200000:1.0:1788201705.332265:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.332267:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.332285:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.332287:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.332297:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451112960, count: 4096 00000080:00200000:1.0:1788201705.332303:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451112960, bytes: 4096 00000080:00200000:1.0:1788201705.332304:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.332306:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.332307:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451112960, 451117056) 0 0 00000080:00200000:1.0:1788201705.332309:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.332312:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110135, 110135] 00020000:00200000:1.0:1788201705.332314:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451112960 chunk: [451112960, 451117056) 451117056, 4096 00020000:00200000:1.0:1788201705.332315:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.332322:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110135, 110135] 00000008:00010000:1.0:1788201705.332325:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.332326:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.332331:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332336:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451112960 451117055) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332342:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332354:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332361:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.332369:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451112960, 451117056) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.332374:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110135 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.332379:0:17463:0:(rw26.c:898:ll_write_end()) pos 451112960, len 4096, copied 4096 00000080:00200000:1.0:1788201705.332383:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a87e500a[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110135] 00000080:00200000:1.0:1788201705.332386:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006b2cab4b fffffc0001029 2:0 ffff93c0c10f2380 110135 lru 00000080:00200000:1.0:1788201705.332392:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000864477aa 110135: 1< 2 - - > 2< 451112960 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61720 - -> 00000080:00200000:1.0:1788201705.332394:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a87e500a 00000080:00200000:1.0:1788201705.332395:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.332397:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.332403:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.332411:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.332413:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.332415:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.332421:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.332423:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.332424:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.332427:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332435:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.332440:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.332441:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.332444:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.332445:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110135, 110135] 00000080:00200000:1.0:1788201705.332453:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.332454:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.332472:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.332475:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.332485:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451117056, count: 4096 00000080:00200000:1.0:1788201705.332491:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451117056, bytes: 4096 00000080:00200000:1.0:1788201705.332492:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.332495:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.332496:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451117056, 451121152) 0 0 00000080:00200000:1.0:1788201705.332497:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.332500:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110136, 110136] 00020000:00200000:1.0:1788201705.332502:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451117056 chunk: [451117056, 451121152) 451121152, 4096 00020000:00200000:1.0:1788201705.332504:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.332510:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110136, 110136] 00000008:00010000:1.0:1788201705.332513:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.332515:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.332520:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332526:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451117056 451121151) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332535:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332539:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332543:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.332547:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451117056, 451121152) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.332550:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110136 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.332554:0:17463:0:(rw26.c:898:ll_write_end()) pos 451117056, len 4096, copied 4096 00000080:00200000:1.0:1788201705.332556:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004f7ef189[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110136] 00000080:00200000:1.0:1788201705.332558:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c6da67ff fffffc0001029 2:0 ffff93c0c10f28c0 110136 lru 00000080:00200000:1.0:1788201705.332562:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000013be6d59 110136: 1< 2 - - > 2< 451117056 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61721 - -> 00000080:00200000:1.0:1788201705.332563:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004f7ef189 00000080:00200000:1.0:1788201705.332564:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.332565:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.332571:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.332579:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.332580:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.332581:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.332587:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.332589:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.332591:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.332593:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332598:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.332603:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.332604:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.332607:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.332609:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110136, 110136] 00000080:00200000:1.0:1788201705.332615:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.332617:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.332635:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.332637:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.332647:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451121152, count: 4096 00000080:00200000:1.0:1788201705.332653:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451121152, bytes: 4096 00000080:00200000:1.0:1788201705.332655:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.332657:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.332658:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451121152, 451125248) 0 0 00000080:00200000:1.0:1788201705.332659:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.332662:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110137, 110137] 00020000:00200000:1.0:1788201705.332664:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451121152 chunk: [451121152, 451125248) 451125248, 4096 00020000:00200000:1.0:1788201705.332666:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.332672:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110137, 110137] 00000008:00010000:1.0:1788201705.332675:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.332676:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.332681:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332690:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451121152 451125247) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332695:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332700:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332704:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.332707:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451121152, 451125248) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.332710:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110137 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.332713:0:17463:0:(rw26.c:898:ll_write_end()) pos 451121152, len 4096, copied 4096 00000080:00200000:1.0:1788201705.332716:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000dd61c519[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110137] 00000080:00200000:1.0:1788201705.332718:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fba40116 fffffc0001029 2:0 ffff93c0c10f2ee0 110137 lru 00000080:00200000:1.0:1788201705.332721:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008ed966c0 110137: 1< 2 - - > 2< 451121152 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61722 - -> 00000080:00200000:1.0:1788201705.332723:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000dd61c519 00000080:00200000:1.0:1788201705.332723:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.332725:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.332730:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.332737:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.332739:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.332740:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.332746:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.332748:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.332750:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.332752:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332757:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.332762:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.332763:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.332765:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.332767:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110137, 110137] 00000080:00200000:1.0:1788201705.332773:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.332775:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.332793:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.332795:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.332805:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451125248, count: 4096 00000080:00200000:1.0:1788201705.332811:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451125248, bytes: 4096 00000080:00200000:1.0:1788201705.332812:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.332815:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.332816:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451125248, 451129344) 0 0 00000080:00200000:1.0:1788201705.332817:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.332820:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110138, 110138] 00020000:00200000:1.0:1788201705.332822:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451125248 chunk: [451125248, 451129344) 451129344, 4096 00020000:00200000:1.0:1788201705.332823:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.332842:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110138, 110138] 00000008:00010000:1.0:1788201705.332846:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.332847:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.332853:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332858:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451125248 451129343) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332864:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332868:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332872:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.332877:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451125248, 451129344) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.332880:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110138 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.332884:0:17463:0:(rw26.c:898:ll_write_end()) pos 451125248, len 4096, copied 4096 00000080:00200000:1.0:1788201705.332886:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009d6185ce[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110138] 00000080:00200000:1.0:1788201705.332888:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004665636c fffffc0001029 2:0 ffff93c0c10f3000 110138 lru 00000080:00200000:1.0:1788201705.332891:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000fc5f58cf 110138: 1< 2 - - > 2< 451125248 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61723 - -> 00000080:00200000:1.0:1788201705.332893:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009d6185ce 00000080:00200000:1.0:1788201705.332894:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.332895:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.332900:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.332908:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.332910:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.332911:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.332917:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.332919:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.332920:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.332923:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.332927:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.332932:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.332934:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.332936:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.332938:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110138, 110138] 00000080:00200000:1.0:1788201705.332945:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.332947:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.332964:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.332966:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.332976:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451129344, count: 4096 00000080:00200000:1.0:1788201705.332982:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451129344, bytes: 4096 00000080:00200000:1.0:1788201705.332984:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.332986:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.332987:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451129344, 451133440) 0 0 00000080:00200000:1.0:1788201705.332988:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.332991:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110139, 110139] 00020000:00200000:1.0:1788201705.332993:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451129344 chunk: [451129344, 451133440) 451133440, 4096 00020000:00200000:1.0:1788201705.332994:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.333001:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110139, 110139] 00000008:00010000:1.0:1788201705.333004:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.333006:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.333011:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333020:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451129344 451133439) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333028:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333036:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333053:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.333062:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451129344, 451133440) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.333067:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110139 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.333072:0:17463:0:(rw26.c:898:ll_write_end()) pos 451129344, len 4096, copied 4096 00000080:00200000:1.0:1788201705.333075:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fbb0f3a9[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110139] 00000080:00200000:1.0:1788201705.333077:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000293ef1f4 fffffc0001029 2:0 ffff93c0c10f3a80 110139 lru 00000080:00200000:1.0:1788201705.333081:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002e0fb970 110139: 1< 2 - - > 2< 451129344 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61724 - -> 00000080:00200000:1.0:1788201705.333082:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fbb0f3a9 00000080:00200000:1.0:1788201705.333083:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.333084:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.333090:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.333102:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.333103:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.333105:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.333111:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.333113:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.333115:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.333117:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333122:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.333127:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.333129:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.333131:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.333133:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110139, 110139] 00000080:00200000:1.0:1788201705.333140:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.333142:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.333160:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.333162:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.333172:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451133440, count: 4096 00000080:00200000:1.0:1788201705.333178:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451133440, bytes: 4096 00000080:00200000:1.0:1788201705.333180:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.333182:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.333183:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451133440, 451137536) 0 0 00000080:00200000:1.0:1788201705.333185:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.333188:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110140, 110140] 00020000:00200000:1.0:1788201705.333190:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451133440 chunk: [451133440, 451137536) 451137536, 4096 00020000:00200000:1.0:1788201705.333192:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.333198:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110140, 110140] 00000008:00010000:1.0:1788201705.333202:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.333203:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.333208:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333214:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451133440 451137535) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333219:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333223:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333227:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.333231:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451133440, 451137536) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.333234:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110140 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.333237:0:17463:0:(rw26.c:898:ll_write_end()) pos 451133440, len 4096, copied 4096 00000080:00200000:1.0:1788201705.333239:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000019d18ba2[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110140] 00000080:00200000:1.0:1788201705.333241:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006c56e2c6 fffffc0001029 2:0 ffff93c0c10f3380 110140 lru 00000080:00200000:1.0:1788201705.333245:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000022f0a8c 110140: 1< 2 - - > 2< 451133440 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61725 - -> 00000080:00200000:1.0:1788201705.333246:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000019d18ba2 00000080:00200000:1.0:1788201705.333246:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.333248:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.333253:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.333261:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.333263:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.333264:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.333269:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.333271:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.333273:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.333275:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333279:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.333284:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.333285:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.333288:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.333289:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110140, 110140] 00000080:00200000:1.0:1788201705.333296:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.333298:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.333315:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.333317:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.333327:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451137536, count: 4096 00000080:00200000:1.0:1788201705.333333:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451137536, bytes: 4096 00000080:00200000:1.0:1788201705.333335:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.333337:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.333338:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451137536, 451141632) 0 0 00000080:00200000:1.0:1788201705.333340:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.333343:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110141, 110141] 00020000:00200000:1.0:1788201705.333345:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451137536 chunk: [451137536, 451141632) 451141632, 4096 00020000:00200000:1.0:1788201705.333347:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.333357:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110141, 110141] 00000008:00010000:1.0:1788201705.333362:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.333365:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.333372:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333381:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451137536 451141631) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333391:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333398:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333402:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.333407:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451137536, 451141632) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.333411:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110141 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.333415:0:17463:0:(rw26.c:898:ll_write_end()) pos 451137536, len 4096, copied 4096 00000080:00200000:1.0:1788201705.333417:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001c01de98[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110141] 00000080:00200000:1.0:1788201705.333419:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000c5b859a fffffc0001029 2:0 ffff93c0c10f38c0 110141 lru 00000080:00200000:1.0:1788201705.333422:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d3b088cc 110141: 1< 2 - - > 2< 451137536 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61726 - -> 00000080:00200000:1.0:1788201705.333425:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001c01de98 00000080:00200000:1.0:1788201705.333425:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.333427:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.333433:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.333440:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.333442:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.333443:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.333449:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.333451:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.333453:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.333455:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333459:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.333465:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.333466:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.333469:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.333471:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110141, 110141] 00000080:00200000:1.0:1788201705.333478:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.333480:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.333497:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.333500:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.333510:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451141632, count: 4096 00000080:00200000:1.0:1788201705.333516:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451141632, bytes: 4096 00000080:00200000:1.0:1788201705.333518:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.333520:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.333521:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451141632, 451145728) 0 0 00000080:00200000:1.0:1788201705.333523:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.333526:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110142, 110142] 00020000:00200000:1.0:1788201705.333528:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451141632 chunk: [451141632, 451145728) 451145728, 4096 00020000:00200000:1.0:1788201705.333529:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.333536:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110142, 110142] 00000008:00010000:1.0:1788201705.333539:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.333540:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.333545:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333551:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451141632 451145727) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333556:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333561:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333564:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.333568:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451141632, 451145728) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.333571:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110142 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.333574:0:17463:0:(rw26.c:898:ll_write_end()) pos 451141632, len 4096, copied 4096 00000080:00200000:1.0:1788201705.333577:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001c509afb[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110142] 00000080:00200000:1.0:1788201705.333579:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000f724236 fffffc0001029 2:0 ffff93c0c10f3ee0 110142 lru 00000080:00200000:1.0:1788201705.333582:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000061d72995 110142: 1< 2 - - > 2< 451141632 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61727 - -> 00000080:00200000:1.0:1788201705.333584:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001c509afb 00000080:00200000:1.0:1788201705.333584:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.333586:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.333591:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.333598:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.333600:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.333601:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.333607:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.333609:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.333610:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.333612:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333617:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.333622:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.333623:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.333626:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.333627:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110142, 110142] 00000080:00200000:1.0:1788201705.333634:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.333636:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.333653:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.333655:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.333665:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451145728, count: 4096 00000080:00200000:1.0:1788201705.333671:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451145728, bytes: 4096 00000080:00200000:1.0:1788201705.333673:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.333675:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.333676:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451145728, 451149824) 0 0 00000080:00200000:1.0:1788201705.333677:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.333680:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110143, 110143] 00020000:00200000:1.0:1788201705.333682:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451145728 chunk: [451145728, 451149824) 451149824, 4096 00020000:00200000:1.0:1788201705.333684:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.333690:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110143, 110143] 00000008:00010000:1.0:1788201705.333693:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.333694:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.333699:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333705:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451145728 451149823) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333710:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333714:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333718:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.333722:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451145728, 451149824) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.333725:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110143 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.333729:0:17463:0:(rw26.c:898:ll_write_end()) pos 451145728, len 4096, copied 4096 00000080:00200000:1.0:1788201705.333731:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d64109cd[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110143] 00000080:00200000:1.0:1788201705.333733:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ef40576a fffffc0001029 2:0 ffff93c0c10f37e0 110143 lru 00000080:00200000:1.0:1788201705.333737:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ee5d825e 110143: 1< 2 - - > 2< 451145728 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61728 - -> 00000080:00200000:1.0:1788201705.333738:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d64109cd 00000080:00200000:1.0:1788201705.333739:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.333740:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.333745:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.333753:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.333755:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.333756:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.333762:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.333764:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.333765:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.333767:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333772:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.333776:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.333778:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.333780:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.333782:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110143, 110143] 00000080:00200000:1.0:1788201705.333789:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.333790:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.333808:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.333810:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.333820:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451149824, count: 4096 00000080:00200000:1.0:1788201705.333826:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451149824, bytes: 4096 00000080:00200000:1.0:1788201705.333827:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.333842:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.333843:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451149824, 451153920) 0 0 00000080:00200000:1.0:1788201705.333845:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.333848:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110144, 110144] 00020000:00200000:1.0:1788201705.333850:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451149824 chunk: [451149824, 451153920) 451153920, 4096 00020000:00200000:1.0:1788201705.333851:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.333859:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110144, 110144] 00000008:00010000:1.0:1788201705.333862:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.333863:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.333868:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333874:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451149824 451153919) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333879:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333884:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333887:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.333892:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451149824, 451153920) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.333896:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110144 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.333899:0:17463:0:(rw26.c:898:ll_write_end()) pos 451149824, len 4096, copied 4096 00000080:00200000:1.0:1788201705.333901:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000040c226f2[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110144] 00000080:00200000:1.0:1788201705.333903:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000079616d4a fffffc0001029 2:0 ffff93c0c10f32a0 110144 lru 00000080:00200000:1.0:1788201705.333907:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000109b8eab 110144: 1< 2 - - > 2< 451149824 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61729 - -> 00000080:00200000:1.0:1788201705.333908:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000040c226f2 00000080:00200000:1.0:1788201705.333909:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.333910:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.333915:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.333923:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.333925:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.333926:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.333932:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.333934:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.333935:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.333938:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.333942:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.333948:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.333949:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.333952:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.333953:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110144, 110144] 00000080:00200000:1.0:1788201705.333960:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.333962:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.333980:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.333982:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.333992:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451153920, count: 4096 00000080:00200000:1.0:1788201705.333997:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451153920, bytes: 4096 00000080:00200000:1.0:1788201705.333999:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.334002:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.334003:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451153920, 451158016) 0 0 00000080:00200000:1.0:1788201705.334004:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.334007:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110145, 110145] 00020000:00200000:1.0:1788201705.334009:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451153920 chunk: [451153920, 451158016) 451158016, 4096 00020000:00200000:1.0:1788201705.334011:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.334020:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110145, 110145] 00000008:00010000:1.0:1788201705.334025:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.334027:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.334034:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334054:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451153920 451158015) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334064:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334072:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334076:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.334085:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451153920, 451158016) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.334089:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110145 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.334093:0:17463:0:(rw26.c:898:ll_write_end()) pos 451153920, len 4096, copied 4096 00000080:00200000:1.0:1788201705.334095:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d294b433[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110145] 00000080:00200000:1.0:1788201705.334097:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a93eab75 fffffc0001029 2:0 ffff93c0c10f3e00 110145 lru 00000080:00200000:1.0:1788201705.334101:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000040cfdf83 110145: 1< 2 - - > 2< 451153920 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61730 - -> 00000080:00200000:1.0:1788201705.334102:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d294b433 00000080:00200000:1.0:1788201705.334103:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.334104:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.334110:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.334118:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.334119:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.334121:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.334127:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.334128:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.334130:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.334132:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334138:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.334143:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.334144:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.334147:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.334149:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110145, 110145] 00000080:00200000:1.0:1788201705.334156:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.334158:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.334176:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.334178:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.334188:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451158016, count: 4096 00000080:00200000:1.0:1788201705.334194:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451158016, bytes: 4096 00000080:00200000:1.0:1788201705.334196:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.334198:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.334199:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451158016, 451162112) 0 0 00000080:00200000:1.0:1788201705.334200:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.334203:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110146, 110146] 00020000:00200000:1.0:1788201705.334205:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451158016 chunk: [451158016, 451162112) 451162112, 4096 00020000:00200000:1.0:1788201705.334207:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.334214:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110146, 110146] 00000008:00010000:1.0:1788201705.334217:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.334218:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.334223:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334229:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451158016 451162111) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334234:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334238:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334246:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.334250:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451158016, 451162112) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.334253:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110146 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.334256:0:17463:0:(rw26.c:898:ll_write_end()) pos 451158016, len 4096, copied 4096 00000080:00200000:1.0:1788201705.334258:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003292c35a[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110146] 00000080:00200000:1.0:1788201705.334260:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a9654e91 fffffc0001029 2:0 ffff93c0c10f31c0 110146 lru 00000080:00200000:1.0:1788201705.334263:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c7d49651 110146: 1< 2 - - > 2< 451158016 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61731 - -> 00000080:00200000:1.0:1788201705.334264:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003292c35a 00000080:00200000:1.0:1788201705.334265:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.334266:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.334272:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.334279:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.334281:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.334282:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.334288:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.334289:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.334291:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.334293:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334298:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.334306:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.334308:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.334310:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.334312:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110146, 110146] 00000080:00200000:1.0:1788201705.334319:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.334321:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.334338:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.334340:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.334352:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451162112, count: 4096 00000080:00200000:1.0:1788201705.334362:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451162112, bytes: 4096 00000080:00200000:1.0:1788201705.334365:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.334368:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.334370:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451162112, 451166208) 0 0 00000080:00200000:1.0:1788201705.334372:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.334377:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110147, 110147] 00020000:00200000:1.0:1788201705.334380:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451162112 chunk: [451162112, 451166208) 451166208, 4096 00020000:00200000:1.0:1788201705.334382:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.334394:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110147, 110147] 00000008:00010000:1.0:1788201705.334399:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.334400:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.334406:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334412:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451162112 451166207) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334418:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334426:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334430:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.334435:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451162112, 451166208) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.334438:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110147 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.334441:0:17463:0:(rw26.c:898:ll_write_end()) pos 451162112, len 4096, copied 4096 00000080:00200000:1.0:1788201705.334444:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000039f5cdfc[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110147] 00000080:00200000:1.0:1788201705.334445:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000091456c33 fffffc0001029 2:0 ffff93c0c10f3620 110147 lru 00000080:00200000:1.0:1788201705.334449:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f9834860 110147: 1< 2 - - > 2< 451162112 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61732 - -> 00000080:00200000:1.0:1788201705.334450:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000039f5cdfc 00000080:00200000:1.0:1788201705.334450:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.334452:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.334457:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.334465:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.334467:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.334468:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.334474:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.334476:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.334477:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.334480:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334488:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.334493:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.334494:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.334496:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.334498:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110147, 110147] 00000080:00200000:1.0:1788201705.334505:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.334507:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.334525:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.334527:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.334537:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451166208, count: 4096 00000080:00200000:1.0:1788201705.334543:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451166208, bytes: 4096 00000080:00200000:1.0:1788201705.334545:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.334547:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.334548:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451166208, 451170304) 0 0 00000080:00200000:1.0:1788201705.334549:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.334553:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110148, 110148] 00020000:00200000:1.0:1788201705.334554:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451166208 chunk: [451166208, 451170304) 451170304, 4096 00020000:00200000:1.0:1788201705.334556:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.334562:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110148, 110148] 00000008:00010000:1.0:1788201705.334565:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.334567:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.334572:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334577:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451166208 451170303) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334586:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334590:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334594:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.334599:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451166208, 451170304) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.334601:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110148 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.334605:0:17463:0:(rw26.c:898:ll_write_end()) pos 451166208, len 4096, copied 4096 00000080:00200000:1.0:1788201705.334607:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d08d9825[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110148] 00000080:00200000:1.0:1788201705.334609:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000daccfdd2 fffffc0001029 2:0 ffff93c0c10f3d20 110148 lru 00000080:00200000:1.0:1788201705.334613:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000094e8594f 110148: 1< 2 - - > 2< 451166208 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61733 - -> 00000080:00200000:1.0:1788201705.334614:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d08d9825 00000080:00200000:1.0:1788201705.334615:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.334616:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.334621:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.334629:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.334631:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.334632:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.334638:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.334640:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.334642:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.334644:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334649:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.334653:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.334655:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.334657:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.334659:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110148, 110148] 00000080:00200000:1.0:1788201705.334666:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.334668:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.334685:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.334687:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.334697:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451170304, count: 4096 00000080:00200000:1.0:1788201705.334703:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451170304, bytes: 4096 00000080:00200000:1.0:1788201705.334705:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.334707:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.334708:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451170304, 451174400) 0 0 00000080:00200000:1.0:1788201705.334709:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.334712:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110149, 110149] 00020000:00200000:1.0:1788201705.334714:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451170304 chunk: [451170304, 451174400) 451174400, 4096 00020000:00200000:1.0:1788201705.334715:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.334722:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110149, 110149] 00000008:00010000:1.0:1788201705.334725:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.334726:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.334731:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334740:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451170304 451174399) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334744:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334749:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334753:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.334757:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451170304, 451174400) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.334760:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110149 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.334763:0:17463:0:(rw26.c:898:ll_write_end()) pos 451170304, len 4096, copied 4096 00000080:00200000:1.0:1788201705.334765:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008fb64e27[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110149] 00000080:00200000:1.0:1788201705.334767:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ec5aaf18 fffffc0001029 2:0 ffff93c0c10f39a0 110149 lru 00000080:00200000:1.0:1788201705.334770:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e2e63375 110149: 1< 2 - - > 2< 451170304 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61734 - -> 00000080:00200000:1.0:1788201705.334771:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008fb64e27 00000080:00200000:1.0:1788201705.334772:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.334773:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.334779:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.334787:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.334788:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.334790:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.334796:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.334797:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.334799:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.334801:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334806:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.334811:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.334812:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.334815:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.334816:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110149, 110149] 00000080:00200000:1.0:1788201705.334823:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.334825:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.334861:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.334863:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.334873:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451174400, count: 4096 00000080:00200000:1.0:1788201705.334879:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451174400, bytes: 4096 00000080:00200000:1.0:1788201705.334881:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.334884:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.334885:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451174400, 451178496) 0 0 00000080:00200000:1.0:1788201705.334886:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.334890:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110150, 110150] 00020000:00200000:1.0:1788201705.334892:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451174400 chunk: [451174400, 451178496) 451178496, 4096 00020000:00200000:1.0:1788201705.334894:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.334901:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110150, 110150] 00000008:00010000:1.0:1788201705.334904:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.334905:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.334911:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334916:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451174400 451178495) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334922:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334926:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334930:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.334935:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451174400, 451178496) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.334938:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110150 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.334941:0:17463:0:(rw26.c:898:ll_write_end()) pos 451174400, len 4096, copied 4096 00000080:00200000:1.0:1788201705.334944:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000b171339[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110150] 00000080:00200000:1.0:1788201705.334946:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f7b89fdc fffffc0001029 2:0 ffff93c0c10bc2a0 110150 lru 00000080:00200000:1.0:1788201705.334949:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b22b841b 110150: 1< 2 - - > 2< 451174400 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61735 - -> 00000080:00200000:1.0:1788201705.334951:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000b171339 00000080:00200000:1.0:1788201705.334952:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.334953:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.334958:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.334966:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.334968:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.334969:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.334975:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.334977:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.334978:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.334980:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.334986:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.334990:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.334992:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.334994:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.334996:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110150, 110150] 00000080:00200000:1.0:1788201705.335002:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.335004:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.335028:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.335031:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.335057:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451178496, count: 4096 00000080:00200000:1.0:1788201705.335069:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451178496, bytes: 4096 00000080:00200000:1.0:1788201705.335072:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.335075:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.335076:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451178496, 451182592) 0 0 00000080:00200000:1.0:1788201705.335078:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.335081:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110151, 110151] 00020000:00200000:1.0:1788201705.335083:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451178496 chunk: [451178496, 451182592) 451182592, 4096 00020000:00200000:1.0:1788201705.335085:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.335092:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110151, 110151] 00000008:00010000:1.0:1788201705.335096:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.335097:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.335103:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335108:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451178496 451182591) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335114:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335118:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335122:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.335127:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451178496, 451182592) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.335130:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110151 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.335133:0:17463:0:(rw26.c:898:ll_write_end()) pos 451178496, len 4096, copied 4096 00000080:00200000:1.0:1788201705.335136:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ff612411[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110151] 00000080:00200000:1.0:1788201705.335138:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000031e7dfb9 fffffc0001029 2:0 ffff93c0c10bdee0 110151 lru 00000080:00200000:1.0:1788201705.335141:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000003227a5c 110151: 1< 2 - - > 2< 451178496 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61736 - -> 00000080:00200000:1.0:1788201705.335143:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ff612411 00000080:00200000:1.0:1788201705.335144:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.335145:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.335151:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.335162:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.335164:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.335165:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.335171:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.335173:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.335174:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.335176:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335181:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.335186:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.335187:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.335190:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.335192:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110151, 110151] 00000080:00200000:1.0:1788201705.335198:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.335200:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.335218:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.335220:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.335230:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451182592, count: 4096 00000080:00200000:1.0:1788201705.335236:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451182592, bytes: 4096 00000080:00200000:1.0:1788201705.335238:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.335240:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.335241:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451182592, 451186688) 0 0 00000080:00200000:1.0:1788201705.335243:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.335246:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110152, 110152] 00020000:00200000:1.0:1788201705.335248:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451182592 chunk: [451182592, 451186688) 451186688, 4096 00020000:00200000:1.0:1788201705.335249:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.335256:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110152, 110152] 00000008:00010000:1.0:1788201705.335259:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.335260:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.335265:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335271:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451182592 451186687) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335276:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335281:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335284:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.335288:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451182592, 451186688) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.335291:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110152 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.335294:0:17463:0:(rw26.c:898:ll_write_end()) pos 451182592, len 4096, copied 4096 00000080:00200000:1.0:1788201705.335297:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009d6f8fdf[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110152] 00000080:00200000:1.0:1788201705.335298:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003654d476 fffffc0001029 2:0 ffff93c0c10bd7e0 110152 lru 00000080:00200000:1.0:1788201705.335302:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c0a67c88 110152: 1< 2 - - > 2< 451182592 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61737 - -> 00000080:00200000:1.0:1788201705.335303:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009d6f8fdf 00000080:00200000:1.0:1788201705.335304:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.335305:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.335311:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.335318:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.335320:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.335321:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.335327:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.335328:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.335330:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.335332:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335337:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.335342:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.335343:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.335346:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.335349:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110152, 110152] 00000080:00200000:1.0:1788201705.335358:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.335362:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.335389:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.335393:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.335406:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451186688, count: 4096 00000080:00200000:1.0:1788201705.335412:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451186688, bytes: 4096 00000080:00200000:1.0:1788201705.335414:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.335416:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.335417:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451186688, 451190784) 0 0 00000080:00200000:1.0:1788201705.335419:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.335423:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110153, 110153] 00020000:00200000:1.0:1788201705.335425:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451186688 chunk: [451186688, 451190784) 451190784, 4096 00020000:00200000:1.0:1788201705.335426:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.335433:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110153, 110153] 00000008:00010000:1.0:1788201705.335437:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.335438:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.335443:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335449:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451186688 451190783) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335454:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335459:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335463:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.335467:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451186688, 451190784) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.335470:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110153 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.335474:0:17463:0:(rw26.c:898:ll_write_end()) pos 451186688, len 4096, copied 4096 00000080:00200000:1.0:1788201705.335476:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000030b7f96e[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110153] 00000080:00200000:1.0:1788201705.335478:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000931470ee fffffc0001029 2:0 ffff93c0c10bd2a0 110153 lru 00000080:00200000:1.0:1788201705.335482:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000012c6f3ed 110153: 1< 2 - - > 2< 451186688 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61738 - -> 00000080:00200000:1.0:1788201705.335484:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000030b7f96e 00000080:00200000:1.0:1788201705.335485:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.335486:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.335492:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.335499:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.335501:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.335502:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.335508:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.335510:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.335511:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.335514:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335519:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.335523:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.335525:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.335527:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.335529:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110153, 110153] 00000080:00200000:1.0:1788201705.335535:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.335537:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.335555:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.335557:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.335567:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451190784, count: 4096 00000080:00200000:1.0:1788201705.335573:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451190784, bytes: 4096 00000080:00200000:1.0:1788201705.335575:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.335577:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.335578:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451190784, 451194880) 0 0 00000080:00200000:1.0:1788201705.335579:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.335582:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110154, 110154] 00020000:00200000:1.0:1788201705.335584:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451190784 chunk: [451190784, 451194880) 451194880, 4096 00020000:00200000:1.0:1788201705.335585:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.335592:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110154, 110154] 00000008:00010000:1.0:1788201705.335595:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.335596:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.335601:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335606:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451190784 451194879) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335612:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335616:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335620:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.335624:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451190784, 451194880) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.335627:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110154 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.335630:0:17463:0:(rw26.c:898:ll_write_end()) pos 451190784, len 4096, copied 4096 00000080:00200000:1.0:1788201705.335632:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006ee5bad6[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110154] 00000080:00200000:1.0:1788201705.335634:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008e6fa421 fffffc0001029 2:0 ffff93c0c10bde00 110154 lru 00000080:00200000:1.0:1788201705.335637:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000dbe5764c 110154: 1< 2 - - > 2< 451190784 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61739 - -> 00000080:00200000:1.0:1788201705.335639:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006ee5bad6 00000080:00200000:1.0:1788201705.335639:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.335641:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.335646:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.335653:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.335654:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.335655:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.335661:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.335663:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.335664:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.335667:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335671:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.335676:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.335677:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.335680:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.335682:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110154, 110154] 00000080:00200000:1.0:1788201705.335688:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.335690:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.335707:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.335710:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.335720:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451194880, count: 4096 00000080:00200000:1.0:1788201705.335726:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451194880, bytes: 4096 00000080:00200000:1.0:1788201705.335727:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.335730:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.335731:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451194880, 451198976) 0 0 00000080:00200000:1.0:1788201705.335732:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.335735:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110155, 110155] 00020000:00200000:1.0:1788201705.335737:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451194880 chunk: [451194880, 451198976) 451198976, 4096 00020000:00200000:1.0:1788201705.335738:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.335745:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110155, 110155] 00000008:00010000:1.0:1788201705.335748:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.335749:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.335754:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335759:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451194880 451198975) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335765:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335769:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335773:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.335777:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451194880, 451198976) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.335780:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110155 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.335783:0:17463:0:(rw26.c:898:ll_write_end()) pos 451194880, len 4096, copied 4096 00000080:00200000:1.0:1788201705.335786:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000048d4b220[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110155] 00000080:00200000:1.0:1788201705.335787:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f7b89d7a fffffc0001029 2:0 ffff93c0c10bd1c0 110155 lru 00000080:00200000:1.0:1788201705.335791:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009733bfd4 110155: 1< 2 - - > 2< 451194880 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61740 - -> 00000080:00200000:1.0:1788201705.335792:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000048d4b220 00000080:00200000:1.0:1788201705.335793:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.335794:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.335800:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.335807:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.335809:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.335810:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.335816:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.335818:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.335819:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.335822:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335827:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.335844:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.335846:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.335849:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.335851:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110155, 110155] 00000080:00200000:1.0:1788201705.335857:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.335859:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.335877:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.335880:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.335889:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451198976, count: 4096 00000080:00200000:1.0:1788201705.335895:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451198976, bytes: 4096 00000080:00200000:1.0:1788201705.335897:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.335899:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.335900:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451198976, 451203072) 0 0 00000080:00200000:1.0:1788201705.335902:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.335905:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110156, 110156] 00020000:00200000:1.0:1788201705.335907:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451198976 chunk: [451198976, 451203072) 451203072, 4096 00020000:00200000:1.0:1788201705.335908:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.335914:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110156, 110156] 00000008:00010000:1.0:1788201705.335918:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.335919:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.335924:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335930:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451198976 451203071) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335935:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335940:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335943:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.335948:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451198976, 451203072) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.335951:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110156 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.335954:0:17463:0:(rw26.c:898:ll_write_end()) pos 451198976, len 4096, copied 4096 00000080:00200000:1.0:1788201705.335957:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000033018585[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110156] 00000080:00200000:1.0:1788201705.335958:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e02d1eaa fffffc0001029 2:0 ffff93c0c10bd620 110156 lru 00000080:00200000:1.0:1788201705.335962:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e4695915 110156: 1< 2 - - > 2< 451198976 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61741 - -> 00000080:00200000:1.0:1788201705.335963:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000033018585 00000080:00200000:1.0:1788201705.335963:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.335965:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.335970:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.335977:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.335979:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.335980:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.335986:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.335988:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.335989:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.335992:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.335996:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.336000:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.336002:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.336005:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.336007:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110156, 110156] 00000080:00200000:1.0:1788201705.336015:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.336018:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.336056:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.336060:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.336075:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451203072, count: 4096 00000080:00200000:1.0:1788201705.336083:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451203072, bytes: 4096 00000080:00200000:1.0:1788201705.336085:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.336087:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.336088:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451203072, 451207168) 0 0 00000080:00200000:1.0:1788201705.336090:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.336093:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110157, 110157] 00020000:00200000:1.0:1788201705.336095:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451203072 chunk: [451203072, 451207168) 451207168, 4096 00020000:00200000:1.0:1788201705.336097:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.336104:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110157, 110157] 00000008:00010000:1.0:1788201705.336107:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.336108:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.336114:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336119:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451203072 451207167) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336125:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336129:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336133:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.336141:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451203072, 451207168) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.336144:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110157 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.336148:0:17463:0:(rw26.c:898:ll_write_end()) pos 451203072, len 4096, copied 4096 00000080:00200000:1.0:1788201705.336150:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008736383e[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110157] 00000080:00200000:1.0:1788201705.336152:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d4652796 fffffc0001029 2:0 ffff93c0c10bdd20 110157 lru 00000080:00200000:1.0:1788201705.336156:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000500a23c3 110157: 1< 2 - - > 2< 451203072 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61742 - -> 00000080:00200000:1.0:1788201705.336157:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008736383e 00000080:00200000:1.0:1788201705.336158:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.336159:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.336164:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.336172:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.336174:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.336175:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.336181:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.336183:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.336184:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.336187:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336191:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.336197:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.336198:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.336201:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.336203:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110157, 110157] 00000080:00200000:1.0:1788201705.336209:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.336211:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.336229:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.336231:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.336241:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451207168, count: 4096 00000080:00200000:1.0:1788201705.336246:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451207168, bytes: 4096 00000080:00200000:1.0:1788201705.336248:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.336251:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.336252:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451207168, 451211264) 0 0 00000080:00200000:1.0:1788201705.336253:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.336256:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110158, 110158] 00020000:00200000:1.0:1788201705.336258:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451207168 chunk: [451207168, 451211264) 451211264, 4096 00020000:00200000:1.0:1788201705.336260:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.336266:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110158, 110158] 00000008:00010000:1.0:1788201705.336269:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.336270:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.336275:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336281:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451207168 451211263) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336286:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336290:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336298:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.336302:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451207168, 451211264) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.336305:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110158 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.336308:0:17463:0:(rw26.c:898:ll_write_end()) pos 451207168, len 4096, copied 4096 00000080:00200000:1.0:1788201705.336310:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000db4dcd02[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110158] 00000080:00200000:1.0:1788201705.336312:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c417cc99 fffffc0001029 2:0 ffff93c0c10bd9a0 110158 lru 00000080:00200000:1.0:1788201705.336316:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b4c759f1 110158: 1< 2 - - > 2< 451207168 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61743 - -> 00000080:00200000:1.0:1788201705.336317:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000db4dcd02 00000080:00200000:1.0:1788201705.336318:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.336319:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.336324:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.336331:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.336333:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.336334:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.336340:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.336342:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.336343:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.336346:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336354:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.336367:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.336369:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.336373:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.336376:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110158, 110158] 00000080:00200000:1.0:1788201705.336387:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.336390:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.336411:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.336413:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.336424:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451211264, count: 4096 00000080:00200000:1.0:1788201705.336429:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451211264, bytes: 4096 00000080:00200000:1.0:1788201705.336431:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.336434:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.336435:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451211264, 451215360) 0 0 00000080:00200000:1.0:1788201705.336436:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.336440:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110159, 110159] 00020000:00200000:1.0:1788201705.336442:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451211264 chunk: [451211264, 451215360) 451215360, 4096 00020000:00200000:1.0:1788201705.336443:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.336450:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110159, 110159] 00000008:00010000:1.0:1788201705.336453:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.336454:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.336459:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336465:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451211264 451215359) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336471:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336480:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336484:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.336488:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451211264, 451215360) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.336491:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110159 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.336494:0:17463:0:(rw26.c:898:ll_write_end()) pos 451211264, len 4096, copied 4096 00000080:00200000:1.0:1788201705.336496:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ecc43ecf[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110159] 00000080:00200000:1.0:1788201705.336498:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000091c37be3 fffffc0001029 2:0 ffff93c0c10bdb60 110159 lru 00000080:00200000:1.0:1788201705.336501:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d9407858 110159: 1< 2 - - > 2< 451211264 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61744 - -> 00000080:00200000:1.0:1788201705.336502:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ecc43ecf 00000080:00200000:1.0:1788201705.336503:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.336504:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.336510:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.336518:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.336519:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.336521:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.336527:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.336528:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.336530:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.336532:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336540:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.336545:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.336546:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.336549:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.336550:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110159, 110159] 00000080:00200000:1.0:1788201705.336557:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.336559:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.336576:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.336578:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.336588:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451215360, count: 4096 00000080:00200000:1.0:1788201705.336594:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451215360, bytes: 4096 00000080:00200000:1.0:1788201705.336595:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.336598:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.336599:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451215360, 451219456) 0 0 00000080:00200000:1.0:1788201705.336600:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.336603:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110160, 110160] 00020000:00200000:1.0:1788201705.336605:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451215360 chunk: [451215360, 451219456) 451219456, 4096 00020000:00200000:1.0:1788201705.336607:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.336613:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110160, 110160] 00000008:00010000:1.0:1788201705.336616:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.336617:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.336622:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336628:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451215360 451219455) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336636:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336641:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336644:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.336649:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451215360, 451219456) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.336652:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110160 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.336655:0:17463:0:(rw26.c:898:ll_write_end()) pos 451215360, len 4096, copied 4096 00000080:00200000:1.0:1788201705.336658:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e5fb9cbb[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110160] 00000080:00200000:1.0:1788201705.336660:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002052015b fffffc0001029 2:0 ffff93c0c10bdc40 110160 lru 00000080:00200000:1.0:1788201705.336663:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001ca7e37b 110160: 1< 2 - - > 2< 451215360 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61745 - -> 00000080:00200000:1.0:1788201705.336664:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e5fb9cbb 00000080:00200000:1.0:1788201705.336665:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.336666:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.336672:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.336679:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.336681:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.336682:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.336688:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.336689:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.336691:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.336693:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336698:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.336703:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.336704:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.336707:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.336709:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110160, 110160] 00000080:00200000:1.0:1788201705.336715:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.336717:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.336734:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.336736:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.336746:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451219456, count: 4096 00000080:00200000:1.0:1788201705.336752:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451219456, bytes: 4096 00000080:00200000:1.0:1788201705.336754:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.336756:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.336757:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451219456, 451223552) 0 0 00000080:00200000:1.0:1788201705.336758:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.336761:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110161, 110161] 00020000:00200000:1.0:1788201705.336763:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451219456 chunk: [451219456, 451223552) 451223552, 4096 00020000:00200000:1.0:1788201705.336764:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.336771:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110161, 110161] 00000008:00010000:1.0:1788201705.336774:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.336775:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.336780:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336789:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451219456 451223551) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336793:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336798:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336802:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.336806:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451219456, 451223552) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.336809:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110161 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.336812:0:17463:0:(rw26.c:898:ll_write_end()) pos 451219456, len 4096, copied 4096 00000080:00200000:1.0:1788201705.336814:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005e0cce16[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110161] 00000080:00200000:1.0:1788201705.336816:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d6024578 fffffc0001029 2:0 ffff93c0c10bd540 110161 lru 00000080:00200000:1.0:1788201705.336819:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001d781e3f 110161: 1< 2 - - > 2< 451219456 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61746 - -> 00000080:00200000:1.0:1788201705.336820:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005e0cce16 00000080:00200000:1.0:1788201705.336821:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.336822:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.336828:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.336848:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.336851:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.336852:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.336858:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.336860:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.336861:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.336864:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336869:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.336874:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.336875:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.336877:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.336879:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110161, 110161] 00000080:00200000:1.0:1788201705.336886:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.336887:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.336905:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.336907:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.336917:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451223552, count: 4096 00000080:00200000:1.0:1788201705.336923:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451223552, bytes: 4096 00000080:00200000:1.0:1788201705.336925:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.336927:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.336928:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451223552, 451227648) 0 0 00000080:00200000:1.0:1788201705.336929:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.336932:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110162, 110162] 00020000:00200000:1.0:1788201705.336934:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451223552 chunk: [451223552, 451227648) 451227648, 4096 00020000:00200000:1.0:1788201705.336936:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.336942:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110162, 110162] 00000008:00010000:1.0:1788201705.336945:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.336947:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.336952:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336958:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451223552 451227647) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336963:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336967:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.336971:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.336976:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451223552, 451227648) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.336979:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110162 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.336982:0:17463:0:(rw26.c:898:ll_write_end()) pos 451223552, len 4096, copied 4096 00000080:00200000:1.0:1788201705.336985:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000de90653e[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110162] 00000080:00200000:1.0:1788201705.336987:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d0f0adfe fffffc0001029 2:0 ffff93c0c10bd460 110162 lru 00000080:00200000:1.0:1788201705.336990:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000278899f3 110162: 1< 2 - - > 2< 451223552 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61747 - -> 00000080:00200000:1.0:1788201705.336992:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000de90653e 00000080:00200000:1.0:1788201705.336992:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.336994:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.336999:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.337006:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.337008:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.337009:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.337017:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.337019:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.337022:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.337025:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337033:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.337052:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.337055:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.337059:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.337062:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110162, 110162] 00000080:00200000:1.0:1788201705.337073:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.337075:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.337094:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.337097:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.337107:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451227648, count: 4096 00000080:00200000:1.0:1788201705.337112:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451227648, bytes: 4096 00000080:00200000:1.0:1788201705.337114:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.337116:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.337118:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451227648, 451231744) 0 0 00000080:00200000:1.0:1788201705.337119:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.337122:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110163, 110163] 00020000:00200000:1.0:1788201705.337124:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451227648 chunk: [451227648, 451231744) 451231744, 4096 00020000:00200000:1.0:1788201705.337126:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.337133:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110163, 110163] 00000008:00010000:1.0:1788201705.337136:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.337137:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.337143:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337148:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451227648 451231743) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337154:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337158:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337162:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.337167:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451227648, 451231744) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.337170:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110163 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.337174:0:17463:0:(rw26.c:898:ll_write_end()) pos 451227648, len 4096, copied 4096 00000080:00200000:1.0:1788201705.337176:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b3a44c22[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110163] 00000080:00200000:1.0:1788201705.337178:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000063d42601 fffffc0001029 2:0 ffff93c0c10bd0e0 110163 lru 00000080:00200000:1.0:1788201705.337182:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000aa4c9c96 110163: 1< 2 - - > 2< 451227648 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61748 - -> 00000080:00200000:1.0:1788201705.337183:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b3a44c22 00000080:00200000:1.0:1788201705.337184:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.337185:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.337191:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.337202:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.337203:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.337205:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.337211:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.337212:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.337214:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.337216:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337221:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.337226:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.337228:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.337230:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.337232:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110163, 110163] 00000080:00200000:1.0:1788201705.337239:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.337241:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.337258:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.337260:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.337270:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451231744, count: 4096 00000080:00200000:1.0:1788201705.337276:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451231744, bytes: 4096 00000080:00200000:1.0:1788201705.337278:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.337280:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.337281:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451231744, 451235840) 0 0 00000080:00200000:1.0:1788201705.337283:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.337286:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110164, 110164] 00020000:00200000:1.0:1788201705.337288:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451231744 chunk: [451231744, 451235840) 451235840, 4096 00020000:00200000:1.0:1788201705.337289:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.337296:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110164, 110164] 00000008:00010000:1.0:1788201705.337298:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.337300:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.337305:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337310:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451231744 451235839) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337316:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337320:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337324:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.337328:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451231744, 451235840) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.337331:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110164 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.337334:0:17463:0:(rw26.c:898:ll_write_end()) pos 451231744, len 4096, copied 4096 00000080:00200000:1.0:1788201705.337336:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000025d471a6[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110164] 00000080:00200000:1.0:1788201705.337338:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000042a908df fffffc0001029 2:0 ffff93c0c10bd700 110164 lru 00000080:00200000:1.0:1788201705.337341:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ab4a47da 110164: 1< 2 - - > 2< 451231744 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61749 - -> 00000080:00200000:1.0:1788201705.337343:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000025d471a6 00000080:00200000:1.0:1788201705.337343:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.337345:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.337353:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.337364:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.337367:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.337369:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.337378:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.337381:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.337383:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.337388:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337396:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.337402:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.337404:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.337407:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.337409:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110164, 110164] 00000080:00200000:1.0:1788201705.337416:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.337418:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.337437:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.337439:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.337449:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451235840, count: 4096 00000080:00200000:1.0:1788201705.337455:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451235840, bytes: 4096 00000080:00200000:1.0:1788201705.337457:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.337459:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.337460:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451235840, 451239936) 0 0 00000080:00200000:1.0:1788201705.337462:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.337465:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110165, 110165] 00020000:00200000:1.0:1788201705.337467:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451235840 chunk: [451235840, 451239936) 451239936, 4096 00020000:00200000:1.0:1788201705.337469:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.337475:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110165, 110165] 00000008:00010000:1.0:1788201705.337478:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.337480:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.337485:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337490:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451235840 451239935) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337496:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337500:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337504:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.337508:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451235840, 451239936) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.337511:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110165 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.337515:0:17463:0:(rw26.c:898:ll_write_end()) pos 451235840, len 4096, copied 4096 00000080:00200000:1.0:1788201705.337517:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000429d68d1[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110165] 00000080:00200000:1.0:1788201705.337518:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000027849159 fffffc0001029 2:0 ffff93c0c10bd000 110165 lru 00000080:00200000:1.0:1788201705.337522:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000054a661e2 110165: 1< 2 - - > 2< 451235840 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61750 - -> 00000080:00200000:1.0:1788201705.337524:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000429d68d1 00000080:00200000:1.0:1788201705.337525:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.337526:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.337532:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.337539:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.337541:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.337542:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.337548:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.337550:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.337551:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.337554:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337558:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.337563:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.337565:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.337567:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.337569:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110165, 110165] 00000080:00200000:1.0:1788201705.337576:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.337578:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.337595:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.337597:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.337607:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451239936, count: 4096 00000080:00200000:1.0:1788201705.337613:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451239936, bytes: 4096 00000080:00200000:1.0:1788201705.337615:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.337617:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.337618:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451239936, 451244032) 0 0 00000080:00200000:1.0:1788201705.337619:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.337622:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110166, 110166] 00020000:00200000:1.0:1788201705.337624:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451239936 chunk: [451239936, 451244032) 451244032, 4096 00020000:00200000:1.0:1788201705.337625:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.337632:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110166, 110166] 00000008:00010000:1.0:1788201705.337635:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.337636:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.337641:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337647:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451239936 451244031) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337652:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337657:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337660:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.337664:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451239936, 451244032) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.337667:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110166 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.337670:0:17463:0:(rw26.c:898:ll_write_end()) pos 451239936, len 4096, copied 4096 00000080:00200000:1.0:1788201705.337673:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000232e99ed[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110166] 00000080:00200000:1.0:1788201705.337675:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000010adedc3 fffffc0001029 2:0 ffff93c0c10bda80 110166 lru 00000080:00200000:1.0:1788201705.337678:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ac4bfacc 110166: 1< 2 - - > 2< 451239936 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61751 - -> 00000080:00200000:1.0:1788201705.337680:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000232e99ed 00000080:00200000:1.0:1788201705.337680:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.337682:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.337687:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.337694:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.337696:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.337697:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.337703:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.337704:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.337706:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.337708:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337713:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.337718:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.337719:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.337721:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.337723:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110166, 110166] 00000080:00200000:1.0:1788201705.337730:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.337731:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.337748:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.337750:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.337760:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451244032, count: 4096 00000080:00200000:1.0:1788201705.337766:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451244032, bytes: 4096 00000080:00200000:1.0:1788201705.337768:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.337770:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.337771:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451244032, 451248128) 0 0 00000080:00200000:1.0:1788201705.337773:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.337776:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110167, 110167] 00020000:00200000:1.0:1788201705.337778:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451244032 chunk: [451244032, 451248128) 451248128, 4096 00020000:00200000:1.0:1788201705.337779:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.337786:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110167, 110167] 00000008:00010000:1.0:1788201705.337789:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.337791:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.337795:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337801:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451244032 451248127) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337807:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337811:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337815:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.337819:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451244032, 451248128) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.337822:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110167 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.337825:0:17463:0:(rw26.c:898:ll_write_end()) pos 451244032, len 4096, copied 4096 00000080:00200000:1.0:1788201705.337828:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a68562ae[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110167] 00000080:00200000:1.0:1788201705.337842:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d3757ad2 fffffc0001029 2:0 ffff93c0c10bd380 110167 lru 00000080:00200000:1.0:1788201705.337846:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001ba63285 110167: 1< 2 - - > 2< 451244032 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61752 - -> 00000080:00200000:1.0:1788201705.337847:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a68562ae 00000080:00200000:1.0:1788201705.337848:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.337849:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.337855:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.337862:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.337864:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.337865:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.337871:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.337873:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.337874:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.337877:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337881:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.337886:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.337887:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.337890:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.337892:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110167, 110167] 00000080:00200000:1.0:1788201705.337898:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.337900:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.337918:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.337920:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.337930:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451248128, count: 4096 00000080:00200000:1.0:1788201705.337935:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451248128, bytes: 4096 00000080:00200000:1.0:1788201705.337937:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.337939:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.337940:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451248128, 451252224) 0 0 00000080:00200000:1.0:1788201705.337942:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.337945:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110168, 110168] 00020000:00200000:1.0:1788201705.337947:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451248128 chunk: [451248128, 451252224) 451252224, 4096 00020000:00200000:1.0:1788201705.337949:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.337955:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110168, 110168] 00000008:00010000:1.0:1788201705.337958:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.337959:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.337964:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337970:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451248128 451252223) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337975:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337980:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.337984:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.337988:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451248128, 451252224) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.337991:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110168 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.337995:0:17463:0:(rw26.c:898:ll_write_end()) pos 451248128, len 4096, copied 4096 00000080:00200000:1.0:1788201705.337997:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005a4c31c4[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110168] 00000080:00200000:1.0:1788201705.337999:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008f5ecb73 fffffc0001029 2:0 ffff93c0c10bd8c0 110168 lru 00000080:00200000:1.0:1788201705.338003:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003347be30 110168: 1< 2 - - > 2< 451248128 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61753 - -> 00000080:00200000:1.0:1788201705.338005:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005a4c31c4 00000080:00200000:1.0:1788201705.338006:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.338007:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.338013:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.338025:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.338027:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.338029:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.338050:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.338053:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.338055:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.338059:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338068:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.338075:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.338077:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.338080:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.338083:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110168, 110168] 00000080:00200000:1.0:1788201705.338090:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.338092:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.338111:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.338114:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.338124:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451252224, count: 4096 00000080:00200000:1.0:1788201705.338130:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451252224, bytes: 4096 00000080:00200000:1.0:1788201705.338131:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.338134:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.338135:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451252224, 451256320) 0 0 00000080:00200000:1.0:1788201705.338136:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.338139:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110169, 110169] 00020000:00200000:1.0:1788201705.338141:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451252224 chunk: [451252224, 451256320) 451256320, 4096 00020000:00200000:1.0:1788201705.338143:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.338150:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110169, 110169] 00000008:00010000:1.0:1788201705.338153:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.338154:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.338160:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338165:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451252224 451256319) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338171:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338175:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338179:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.338187:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451252224, 451256320) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.338190:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110169 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.338194:0:17463:0:(rw26.c:898:ll_write_end()) pos 451252224, len 4096, copied 4096 00000080:00200000:1.0:1788201705.338196:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000052adfb39[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110169] 00000080:00200000:1.0:1788201705.338198:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000015e4726e fffffc0001029 2:0 ffff93c0c10be700 110169 lru 00000080:00200000:1.0:1788201705.338202:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a2b16cc6 110169: 1< 2 - - > 2< 451252224 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61754 - -> 00000080:00200000:1.0:1788201705.338203:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000052adfb39 00000080:00200000:1.0:1788201705.338204:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.338205:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.338211:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.338218:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.338220:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.338221:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.338227:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.338229:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.338230:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.338233:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338237:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.338242:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.338243:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.338246:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.338248:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110169, 110169] 00000080:00200000:1.0:1788201705.338254:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.338256:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.338274:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.338276:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.338286:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451256320, count: 4096 00000080:00200000:1.0:1788201705.338292:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451256320, bytes: 4096 00000080:00200000:1.0:1788201705.338294:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.338296:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.338297:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451256320, 451260416) 0 0 00000080:00200000:1.0:1788201705.338298:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.338301:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110170, 110170] 00020000:00200000:1.0:1788201705.338303:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451256320 chunk: [451256320, 451260416) 451260416, 4096 00020000:00200000:1.0:1788201705.338305:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.338311:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110170, 110170] 00000008:00010000:1.0:1788201705.338314:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.338316:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.338320:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338326:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451256320 451260415) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338331:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338336:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338343:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.338350:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451256320, 451260416) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.338355:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110170 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.338360:0:17463:0:(rw26.c:898:ll_write_end()) pos 451256320, len 4096, copied 4096 00000080:00200000:1.0:1788201705.338363:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d3115a62[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110170] 00000080:00200000:1.0:1788201705.338366:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003df46041 fffffc0001029 2:0 ffff93c0c10be000 110170 lru 00000080:00200000:1.0:1788201705.338372:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006579dd6a 110170: 1< 2 - - > 2< 451256320 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61755 - -> 00000080:00200000:1.0:1788201705.338374:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d3115a62 00000080:00200000:1.0:1788201705.338375:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.338377:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.338385:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.338397:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.338399:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.338401:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.338408:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.338410:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.338411:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.338414:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338419:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.338428:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.338429:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.338432:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.338434:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110170, 110170] 00000080:00200000:1.0:1788201705.338441:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.338443:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.338461:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.338463:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.338473:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451260416, count: 4096 00000080:00200000:1.0:1788201705.338479:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451260416, bytes: 4096 00000080:00200000:1.0:1788201705.338481:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.338483:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.338484:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451260416, 451264512) 0 0 00000080:00200000:1.0:1788201705.338485:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.338488:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110171, 110171] 00020000:00200000:1.0:1788201705.338490:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451260416 chunk: [451260416, 451264512) 451264512, 4096 00020000:00200000:1.0:1788201705.338492:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.338498:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110171, 110171] 00000008:00010000:1.0:1788201705.338501:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.338503:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.338508:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338514:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451260416 451264511) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338519:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338528:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338532:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.338536:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451260416, 451264512) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.338539:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110171 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.338542:0:17463:0:(rw26.c:898:ll_write_end()) pos 451260416, len 4096, copied 4096 00000080:00200000:1.0:1788201705.338544:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e71c7efe[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110171] 00000080:00200000:1.0:1788201705.338546:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009ac247eb fffffc0001029 2:0 ffff93c0c10bea80 110171 lru 00000080:00200000:1.0:1788201705.338549:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000528c90fb 110171: 1< 2 - - > 2< 451260416 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61756 - -> 00000080:00200000:1.0:1788201705.338550:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e71c7efe 00000080:00200000:1.0:1788201705.338551:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.338552:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.338557:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.338565:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.338567:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.338568:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.338574:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.338576:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.338577:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.338580:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338587:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.338592:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.338593:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.338596:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.338597:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110171, 110171] 00000080:00200000:1.0:1788201705.338604:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.338606:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.338623:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.338625:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.338635:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451264512, count: 4096 00000080:00200000:1.0:1788201705.338641:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451264512, bytes: 4096 00000080:00200000:1.0:1788201705.338643:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.338645:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.338646:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451264512, 451268608) 0 0 00000080:00200000:1.0:1788201705.338648:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.338651:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110172, 110172] 00020000:00200000:1.0:1788201705.338653:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451264512 chunk: [451264512, 451268608) 451268608, 4096 00020000:00200000:1.0:1788201705.338654:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.338660:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110172, 110172] 00000008:00010000:1.0:1788201705.338663:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.338665:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.338670:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338675:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451264512 451268607) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338684:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338688:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338692:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.338697:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451264512, 451268608) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.338700:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110172 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.338703:0:17463:0:(rw26.c:898:ll_write_end()) pos 451264512, len 4096, copied 4096 00000080:00200000:1.0:1788201705.338706:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000034edb288[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110172] 00000080:00200000:1.0:1788201705.338707:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009b5879fc fffffc0001029 2:0 ffff93c0c10be380 110172 lru 00000080:00200000:1.0:1788201705.338711:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000058bb3392 110172: 1< 2 - - > 2< 451264512 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61757 - -> 00000080:00200000:1.0:1788201705.338712:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000034edb288 00000080:00200000:1.0:1788201705.338713:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.338714:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.338720:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.338727:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.338729:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.338730:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.338736:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.338738:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.338739:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.338742:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338747:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.338753:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.338755:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.338759:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.338762:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110172, 110172] 00000080:00200000:1.0:1788201705.338771:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.338774:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.338802:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.338805:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.338820:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451268608, count: 4096 00000080:00200000:1.0:1788201705.338844:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451268608, bytes: 4096 00000080:00200000:1.0:1788201705.338848:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.338851:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.338852:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451268608, 451272704) 0 0 00000080:00200000:1.0:1788201705.338854:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.338859:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110173, 110173] 00020000:00200000:1.0:1788201705.338862:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451268608 chunk: [451268608, 451272704) 451272704, 4096 00020000:00200000:1.0:1788201705.338864:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.338875:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110173, 110173] 00000008:00010000:1.0:1788201705.338880:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.338882:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.338889:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338901:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451268608 451272703) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338909:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338916:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338923:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.338930:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451268608, 451272704) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.338935:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110173 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.338940:0:17463:0:(rw26.c:898:ll_write_end()) pos 451268608, len 4096, copied 4096 00000080:00200000:1.0:1788201705.338944:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009eaf5c51[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110173] 00000080:00200000:1.0:1788201705.338947:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000785590d8 fffffc0001029 2:0 ffff93c0c10be8c0 110173 lru 00000080:00200000:1.0:1788201705.338952:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000572fe947 110173: 1< 2 - - > 2< 451268608 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61758 - -> 00000080:00200000:1.0:1788201705.338953:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009eaf5c51 00000080:00200000:1.0:1788201705.338954:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.338956:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.338962:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.338970:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.338972:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.338974:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.338981:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.338983:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.338984:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.338987:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.338992:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.338997:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.338998:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.339001:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.339003:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110173, 110173] 00000080:00200000:1.0:1788201705.339010:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.339012:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.339050:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.339054:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.339071:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451272704, count: 4096 00000080:00200000:1.0:1788201705.339080:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451272704, bytes: 4096 00000080:00200000:1.0:1788201705.339082:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.339085:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.339086:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451272704, 451276800) 0 0 00000080:00200000:1.0:1788201705.339087:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.339091:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110174, 110174] 00020000:00200000:1.0:1788201705.339093:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451272704 chunk: [451272704, 451276800) 451276800, 4096 00020000:00200000:1.0:1788201705.339095:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.339102:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110174, 110174] 00000008:00010000:1.0:1788201705.339105:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.339107:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.339113:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339118:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451272704 451276799) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339124:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339128:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339132:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.339137:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451272704, 451276800) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.339140:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110174 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.339144:0:17463:0:(rw26.c:898:ll_write_end()) pos 451272704, len 4096, copied 4096 00000080:00200000:1.0:1788201705.339146:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000086d8dff1[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110174] 00000080:00200000:1.0:1788201705.339148:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008ff41499 fffffc0001029 2:0 ffff93c0c10beee0 110174 lru 00000080:00200000:1.0:1788201705.339152:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000085d3fd8e 110174: 1< 2 - - > 2< 451272704 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61759 - -> 00000080:00200000:1.0:1788201705.339153:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000086d8dff1 00000080:00200000:1.0:1788201705.339154:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.339155:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.339161:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.339169:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.339171:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.339172:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.339179:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.339180:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.339182:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.339184:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339189:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.339194:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.339196:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.339198:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.339200:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110174, 110174] 00000080:00200000:1.0:1788201705.339207:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.339208:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.339226:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.339229:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.339239:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451276800, count: 4096 00000080:00200000:1.0:1788201705.339245:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451276800, bytes: 4096 00000080:00200000:1.0:1788201705.339246:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.339248:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.339249:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451276800, 451280896) 0 0 00000080:00200000:1.0:1788201705.339251:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.339254:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110175, 110175] 00020000:00200000:1.0:1788201705.339256:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451276800 chunk: [451276800, 451280896) 451280896, 4096 00020000:00200000:1.0:1788201705.339257:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.339263:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110175, 110175] 00000008:00010000:1.0:1788201705.339266:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.339268:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.339273:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339278:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451276800 451280895) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339284:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339288:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339292:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.339297:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451276800, 451280896) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.339300:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110175 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.339303:0:17463:0:(rw26.c:898:ll_write_end()) pos 451276800, len 4096, copied 4096 00000080:00200000:1.0:1788201705.339306:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007feaf173[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110175] 00000080:00200000:1.0:1788201705.339307:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000bc1ba7e fffffc0001029 2:0 ffff93c0c10be7e0 110175 lru 00000080:00200000:1.0:1788201705.339311:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cc5f277e 110175: 1< 2 - - > 2< 451276800 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61760 - -> 00000080:00200000:1.0:1788201705.339312:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007feaf173 00000080:00200000:1.0:1788201705.339313:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.339314:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.339320:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.339330:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.339332:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.339334:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.339339:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.339341:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.339342:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.339345:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339353:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.339361:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.339364:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.339368:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.339370:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110175, 110175] 00000080:00200000:1.0:1788201705.339381:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.339384:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.339408:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.339410:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.339421:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451280896, count: 4096 00000080:00200000:1.0:1788201705.339427:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451280896, bytes: 4096 00000080:00200000:1.0:1788201705.339429:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.339431:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.339432:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451280896, 451284992) 0 0 00000080:00200000:1.0:1788201705.339433:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.339437:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110176, 110176] 00020000:00200000:1.0:1788201705.339439:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451280896 chunk: [451280896, 451284992) 451284992, 4096 00020000:00200000:1.0:1788201705.339441:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.339447:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110176, 110176] 00000008:00010000:1.0:1788201705.339450:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.339452:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.339457:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339463:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451280896 451284991) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339468:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339473:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339476:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.339481:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451280896, 451284992) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.339484:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110176 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.339488:0:17463:0:(rw26.c:898:ll_write_end()) pos 451280896, len 4096, copied 4096 00000080:00200000:1.0:1788201705.339490:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e4300c03[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110176] 00000080:00200000:1.0:1788201705.339492:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ff35876c fffffc0001029 2:0 ffff93c0c10be2a0 110176 lru 00000080:00200000:1.0:1788201705.339496:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005d5b3d34 110176: 1< 2 - - > 2< 451280896 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61761 - -> 00000080:00200000:1.0:1788201705.339497:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e4300c03 00000080:00200000:1.0:1788201705.339498:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.339500:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.339505:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.339513:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.339515:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.339516:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.339522:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.339523:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.339525:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.339527:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339532:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.339537:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.339538:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.339540:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.339542:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110176, 110176] 00000080:00200000:1.0:1788201705.339549:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.339551:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.339568:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.339571:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.339580:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451284992, count: 4096 00000080:00200000:1.0:1788201705.339586:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451284992, bytes: 4096 00000080:00200000:1.0:1788201705.339588:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.339590:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.339591:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451284992, 451289088) 0 0 00000080:00200000:1.0:1788201705.339593:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.339596:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110177, 110177] 00020000:00200000:1.0:1788201705.339598:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451284992 chunk: [451284992, 451289088) 451289088, 4096 00020000:00200000:1.0:1788201705.339599:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.339606:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110177, 110177] 00000008:00010000:1.0:1788201705.339609:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.339610:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.339615:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339620:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451284992 451289087) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339626:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339630:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339634:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.339639:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451284992, 451289088) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.339642:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110177 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.339645:0:17463:0:(rw26.c:898:ll_write_end()) pos 451284992, len 4096, copied 4096 00000080:00200000:1.0:1788201705.339647:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000d4b9e7f[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110177] 00000080:00200000:1.0:1788201705.339649:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000096a17d0 fffffc0001029 2:0 ffff93c0c10bee00 110177 lru 00000080:00200000:1.0:1788201705.339653:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000beb28a97 110177: 1< 2 - - > 2< 451284992 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61762 - -> 00000080:00200000:1.0:1788201705.339656:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000d4b9e7f 00000080:00200000:1.0:1788201705.339657:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.339658:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.339663:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.339671:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.339672:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.339674:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.339679:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.339681:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.339683:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.339685:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339689:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.339694:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.339696:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.339698:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.339700:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110177, 110177] 00000080:00200000:1.0:1788201705.339707:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.339709:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.339726:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.339728:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.339738:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451289088, count: 4096 00000080:00200000:1.0:1788201705.339744:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451289088, bytes: 4096 00000080:00200000:1.0:1788201705.339746:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.339748:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.339749:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451289088, 451293184) 0 0 00000080:00200000:1.0:1788201705.339751:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.339754:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110178, 110178] 00020000:00200000:1.0:1788201705.339756:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451289088 chunk: [451289088, 451293184) 451293184, 4096 00020000:00200000:1.0:1788201705.339757:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.339763:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110178, 110178] 00000008:00010000:1.0:1788201705.339766:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.339768:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.339772:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339778:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451289088 451293183) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339783:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339788:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339792:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.339796:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451289088, 451293184) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.339799:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110178 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.339802:0:17463:0:(rw26.c:898:ll_write_end()) pos 451289088, len 4096, copied 4096 00000080:00200000:1.0:1788201705.339805:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005f31fcfb[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110178] 00000080:00200000:1.0:1788201705.339807:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e951f14c fffffc0001029 2:0 ffff93c0c10be1c0 110178 lru 00000080:00200000:1.0:1788201705.339810:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c1967754 110178: 1< 2 - - > 2< 451289088 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61763 - -> 00000080:00200000:1.0:1788201705.339812:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005f31fcfb 00000080:00200000:1.0:1788201705.339812:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.339813:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.339819:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.339826:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.339827:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.339841:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.339848:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.339849:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.339851:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.339853:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339859:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.339863:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.339865:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.339867:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.339869:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110178, 110178] 00000080:00200000:1.0:1788201705.339876:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.339878:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.339895:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.339897:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.339907:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451293184, count: 4096 00000080:00200000:1.0:1788201705.339913:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451293184, bytes: 4096 00000080:00200000:1.0:1788201705.339915:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.339917:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.339918:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451293184, 451297280) 0 0 00000080:00200000:1.0:1788201705.339919:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.339922:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110179, 110179] 00020000:00200000:1.0:1788201705.339924:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451293184 chunk: [451293184, 451297280) 451297280, 4096 00020000:00200000:1.0:1788201705.339926:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.339932:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110179, 110179] 00000008:00010000:1.0:1788201705.339935:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.339937:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.339941:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339947:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451293184 451297279) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339952:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339957:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.339961:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.339964:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451293184, 451297280) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.339967:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110179 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.339971:0:17463:0:(rw26.c:898:ll_write_end()) pos 451293184, len 4096, copied 4096 00000080:00200000:1.0:1788201705.339973:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f62cf7e9[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110179] 00000080:00200000:1.0:1788201705.339975:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000dcc5ad4b fffffc0001029 2:0 ffff93c0c10be620 110179 lru 00000080:00200000:1.0:1788201705.339979:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000039bf295f 110179: 1< 2 - - > 2< 451293184 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61764 - -> 00000080:00200000:1.0:1788201705.339980:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f62cf7e9 00000080:00200000:1.0:1788201705.339981:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.339982:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.339987:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.339995:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.339997:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.339998:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.340004:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.340005:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.340007:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.340009:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340016:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.340025:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.340027:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.340031:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.340034:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110179, 110179] 00000080:00200000:1.0:1788201705.340055:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.340059:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.340083:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.340086:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.340097:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451297280, count: 4096 00000080:00200000:1.0:1788201705.340103:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451297280, bytes: 4096 00000080:00200000:1.0:1788201705.340105:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.340107:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.340108:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451297280, 451301376) 0 0 00000080:00200000:1.0:1788201705.340110:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.340113:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110180, 110180] 00020000:00200000:1.0:1788201705.340115:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451297280 chunk: [451297280, 451301376) 451301376, 4096 00020000:00200000:1.0:1788201705.340117:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.340123:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110180, 110180] 00000008:00010000:1.0:1788201705.340127:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.340128:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.340133:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340139:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451297280 451301375) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340145:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340150:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340154:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.340160:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451297280, 451301376) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.340165:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110180 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.340170:0:17463:0:(rw26.c:898:ll_write_end()) pos 451297280, len 4096, copied 4096 00000080:00200000:1.0:1788201705.340173:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000414159cf[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110180] 00000080:00200000:1.0:1788201705.340176:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004692fada fffffc0001029 2:0 ffff93c0c10bed20 110180 lru 00000080:00200000:1.0:1788201705.340181:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c17515c0 110180: 1< 2 - - > 2< 451297280 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61765 - -> 00000080:00200000:1.0:1788201705.340183:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000414159cf 00000080:00200000:1.0:1788201705.340184:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.340186:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.340194:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.340206:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.340209:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.340212:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.340221:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.340223:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.340226:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.340230:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340237:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.340243:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.340244:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.340248:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.340250:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110180, 110180] 00000080:00200000:1.0:1788201705.340257:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.340259:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.340278:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.340280:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.340290:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451301376, count: 4096 00000080:00200000:1.0:1788201705.340297:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451301376, bytes: 4096 00000080:00200000:1.0:1788201705.340298:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.340301:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.340301:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451301376, 451305472) 0 0 00000080:00200000:1.0:1788201705.340303:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.340306:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110181, 110181] 00020000:00200000:1.0:1788201705.340308:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451301376 chunk: [451301376, 451305472) 451305472, 4096 00020000:00200000:1.0:1788201705.340310:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.340316:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110181, 110181] 00000008:00010000:1.0:1788201705.340319:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.340321:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.340326:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340332:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451301376 451305471) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340337:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340341:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340346:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.340358:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451301376, 451305472) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.340363:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110181 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.340368:0:17463:0:(rw26.c:898:ll_write_end()) pos 451301376, len 4096, copied 4096 00000080:00200000:1.0:1788201705.340372:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000157be535[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110181] 00000080:00200000:1.0:1788201705.340375:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002d9b4b6a fffffc0001029 2:0 ffff93c0c10be9a0 110181 lru 00000080:00200000:1.0:1788201705.340381:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000b6a1d0b 110181: 1< 2 - - > 2< 451301376 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61766 - -> 00000080:00200000:1.0:1788201705.340383:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000157be535 00000080:00200000:1.0:1788201705.340384:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.340386:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.340395:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.340405:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.340407:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.340408:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.340415:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.340416:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.340418:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.340420:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340426:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.340431:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.340432:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.340435:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.340437:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110181, 110181] 00000080:00200000:1.0:1788201705.340444:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.340446:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.340464:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.340466:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.340476:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451305472, count: 4096 00000080:00200000:1.0:1788201705.340482:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451305472, bytes: 4096 00000080:00200000:1.0:1788201705.340484:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.340486:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.340487:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451305472, 451309568) 0 0 00000080:00200000:1.0:1788201705.340488:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.340491:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110182, 110182] 00020000:00200000:1.0:1788201705.340493:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451305472 chunk: [451305472, 451309568) 451309568, 4096 00020000:00200000:1.0:1788201705.340495:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.340501:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110182, 110182] 00000008:00010000:1.0:1788201705.340504:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.340506:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.340511:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340516:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451305472 451309567) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340522:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340526:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340534:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.340538:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451305472, 451309568) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.340541:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110182 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.340544:0:17463:0:(rw26.c:898:ll_write_end()) pos 451305472, len 4096, copied 4096 00000080:00200000:1.0:1788201705.340546:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000036e614da[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110182] 00000080:00200000:1.0:1788201705.340548:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006658bbd4 fffffc0001029 2:0 ffff93c0c10beb60 110182 lru 00000080:00200000:1.0:1788201705.340552:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000dbe63b11 110182: 1< 2 - - > 2< 451305472 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61767 - -> 00000080:00200000:1.0:1788201705.340553:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000036e614da 00000080:00200000:1.0:1788201705.340554:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.340555:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.340561:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.340568:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.340570:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.340571:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.340577:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.340578:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.340580:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.340582:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340587:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.340595:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.340597:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.340599:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.340601:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110182, 110182] 00000080:00200000:1.0:1788201705.340608:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.340610:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.340627:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.340629:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.340639:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451309568, count: 4096 00000080:00200000:1.0:1788201705.340645:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451309568, bytes: 4096 00000080:00200000:1.0:1788201705.340646:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.340649:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.340650:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451309568, 451313664) 0 0 00000080:00200000:1.0:1788201705.340651:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.340654:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110183, 110183] 00020000:00200000:1.0:1788201705.340656:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451309568 chunk: [451309568, 451313664) 451313664, 4096 00020000:00200000:1.0:1788201705.340657:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.340664:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110183, 110183] 00000008:00010000:1.0:1788201705.340667:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.340668:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.340673:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340679:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451309568 451313663) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340684:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340692:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340696:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.340700:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451309568, 451313664) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.340703:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110183 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.340706:0:17463:0:(rw26.c:898:ll_write_end()) pos 451309568, len 4096, copied 4096 00000080:00200000:1.0:1788201705.340708:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b25dd0b3[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110183] 00000080:00200000:1.0:1788201705.340710:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004fe52b8c fffffc0001029 2:0 ffff93c0c10bec40 110183 lru 00000080:00200000:1.0:1788201705.340713:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000090598526 110183: 1< 2 - - > 2< 451309568 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61768 - -> 00000080:00200000:1.0:1788201705.340714:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b25dd0b3 00000080:00200000:1.0:1788201705.340715:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.340716:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.340722:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.340729:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.340731:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.340732:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.340738:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.340739:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.340741:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.340743:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340752:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.340757:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.340758:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.340760:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.340762:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110183, 110183] 00000080:00200000:1.0:1788201705.340769:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.340770:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.340788:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.340790:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.340800:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451313664, count: 4096 00000080:00200000:1.0:1788201705.340806:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451313664, bytes: 4096 00000080:00200000:1.0:1788201705.340808:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.340810:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.340811:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451313664, 451317760) 0 0 00000080:00200000:1.0:1788201705.340812:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.340816:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110184, 110184] 00020000:00200000:1.0:1788201705.340817:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451313664 chunk: [451313664, 451317760) 451317760, 4096 00020000:00200000:1.0:1788201705.340819:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.340825:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110184, 110184] 00000008:00010000:1.0:1788201705.340841:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.340843:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.340848:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340854:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451313664 451317759) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340863:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340867:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340871:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.340875:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451313664, 451317760) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.340878:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110184 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.340882:0:17463:0:(rw26.c:898:ll_write_end()) pos 451313664, len 4096, copied 4096 00000080:00200000:1.0:1788201705.340884:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000068fc33e6[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110184] 00000080:00200000:1.0:1788201705.340886:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a6504dd6 fffffc0001029 2:0 ffff93c0c10be540 110184 lru 00000080:00200000:1.0:1788201705.340889:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000099d603f7 110184: 1< 2 - - > 2< 451313664 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61769 - -> 00000080:00200000:1.0:1788201705.340891:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000068fc33e6 00000080:00200000:1.0:1788201705.340891:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.340892:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.340898:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.340906:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.340907:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.340909:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.340915:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.340916:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.340918:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.340921:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.340926:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.340930:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.340932:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.340934:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.340936:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110184, 110184] 00000080:00200000:1.0:1788201705.340943:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.340945:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.340962:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.340964:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.340973:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451317760, count: 4096 00000080:00200000:1.0:1788201705.340979:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451317760, bytes: 4096 00000080:00200000:1.0:1788201705.340981:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.340983:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.340984:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451317760, 451321856) 0 0 00000080:00200000:1.0:1788201705.340986:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.340989:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110185, 110185] 00020000:00200000:1.0:1788201705.340991:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451317760 chunk: [451317760, 451321856) 451321856, 4096 00020000:00200000:1.0:1788201705.340993:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.340999:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110185, 110185] 00000008:00010000:1.0:1788201705.341002:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.341003:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.341008:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341020:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451317760 451321855) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341028:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341036:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341054:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.341063:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451317760, 451321856) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.341069:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110185 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.341073:0:17463:0:(rw26.c:898:ll_write_end()) pos 451317760, len 4096, copied 4096 00000080:00200000:1.0:1788201705.341076:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000017f106c3[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110185] 00000080:00200000:1.0:1788201705.341078:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b9fc95a2 fffffc0001029 2:0 ffff93c0c10be460 110185 lru 00000080:00200000:1.0:1788201705.341081:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000e484b9e 110185: 1< 2 - - > 2< 451317760 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61770 - -> 00000080:00200000:1.0:1788201705.341083:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000017f106c3 00000080:00200000:1.0:1788201705.341084:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.341085:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.341091:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.341099:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.341101:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.341102:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.341109:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.341111:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.341112:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.341115:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341120:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.341125:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.341126:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.341128:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.341130:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110185, 110185] 00000080:00200000:1.0:1788201705.341137:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.341139:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.341158:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.341160:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.341170:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451321856, count: 4096 00000080:00200000:1.0:1788201705.341176:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451321856, bytes: 4096 00000080:00200000:1.0:1788201705.341178:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.341180:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.341181:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451321856, 451325952) 0 0 00000080:00200000:1.0:1788201705.341182:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.341186:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110186, 110186] 00020000:00200000:1.0:1788201705.341188:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451321856 chunk: [451321856, 451325952) 451325952, 4096 00020000:00200000:1.0:1788201705.341189:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.341196:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110186, 110186] 00000008:00010000:1.0:1788201705.341199:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.341200:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.341206:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341211:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451321856 451325951) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341217:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341221:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341225:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.341229:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451321856, 451325952) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.341232:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110186 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.341235:0:17463:0:(rw26.c:898:ll_write_end()) pos 451321856, len 4096, copied 4096 00000080:00200000:1.0:1788201705.341237:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e8625891[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110186] 00000080:00200000:1.0:1788201705.341239:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c595995b fffffc0001029 2:0 ffff93c0c10be0e0 110186 lru 00000080:00200000:1.0:1788201705.341242:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e89fb939 110186: 1< 2 - - > 2< 451321856 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61771 - -> 00000080:00200000:1.0:1788201705.341244:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e8625891 00000080:00200000:1.0:1788201705.341245:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.341246:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.341251:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.341259:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.341261:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.341262:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.341268:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.341269:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.341271:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.341273:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341278:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.341284:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.341285:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.341287:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.341289:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110186, 110186] 00000080:00200000:1.0:1788201705.341296:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.341297:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.341315:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.341317:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.341327:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451325952, count: 4096 00000080:00200000:1.0:1788201705.341333:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451325952, bytes: 4096 00000080:00200000:1.0:1788201705.341334:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.341336:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.341337:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451325952, 451330048) 0 0 00000080:00200000:1.0:1788201705.341339:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.341342:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110187, 110187] 00020000:00200000:1.0:1788201705.341344:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451325952 chunk: [451325952, 451330048) 451330048, 4096 00020000:00200000:1.0:1788201705.341345:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.341355:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110187, 110187] 00000008:00010000:1.0:1788201705.341360:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.341362:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.341369:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341379:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451325952 451330047) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341388:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341396:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341400:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.341406:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451325952, 451330048) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.341409:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110187 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.341413:0:17463:0:(rw26.c:898:ll_write_end()) pos 451325952, len 4096, copied 4096 00000080:00200000:1.0:1788201705.341415:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000984b6078[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110187] 00000080:00200000:1.0:1788201705.341417:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000262a8600 fffffc0001029 2:0 ffff93c0c11ed380 110187 lru 00000080:00200000:1.0:1788201705.341420:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000033416cf0 110187: 1< 2 - - > 2< 451325952 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61772 - -> 00000080:00200000:1.0:1788201705.341422:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000984b6078 00000080:00200000:1.0:1788201705.341422:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.341424:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.341430:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.341441:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.341442:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.341444:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.341450:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.341452:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.341453:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.341456:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341460:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.341465:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.341467:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.341469:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.341471:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110187, 110187] 00000080:00200000:1.0:1788201705.341478:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.341480:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.341498:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.341500:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.341510:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451330048, count: 4096 00000080:00200000:1.0:1788201705.341516:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451330048, bytes: 4096 00000080:00200000:1.0:1788201705.341518:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.341520:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.341521:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451330048, 451334144) 0 0 00000080:00200000:1.0:1788201705.341522:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.341526:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110188, 110188] 00020000:00200000:1.0:1788201705.341528:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451330048 chunk: [451330048, 451334144) 451334144, 4096 00020000:00200000:1.0:1788201705.341529:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.341536:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110188, 110188] 00000008:00010000:1.0:1788201705.341539:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.341541:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.341546:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341552:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451330048 451334143) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341557:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341561:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341565:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.341569:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451330048, 451334144) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.341572:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110188 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.341575:0:17463:0:(rw26.c:898:ll_write_end()) pos 451330048, len 4096, copied 4096 00000080:00200000:1.0:1788201705.341577:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e3065633[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110188] 00000080:00200000:1.0:1788201705.341579:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000083692dbd fffffc0001029 2:0 ffff93c0c11ed8c0 110188 lru 00000080:00200000:1.0:1788201705.341582:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009c9872d0 110188: 1< 2 - - > 2< 451330048 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61773 - -> 00000080:00200000:1.0:1788201705.341583:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e3065633 00000080:00200000:1.0:1788201705.341584:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.341585:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.341591:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.341598:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.341600:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.341601:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.341607:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.341608:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.341610:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.341612:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341617:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.341621:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.341623:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.341625:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.341627:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110188, 110188] 00000080:00200000:1.0:1788201705.341633:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.341635:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201705.341653:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201705.341655:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201705.341664:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 451334144, count: 4096 00000080:00200000:1.0:1788201705.341670:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 451334144, bytes: 4096 00000080:00200000:1.0:1788201705.341672:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201705.341674:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201705.341675:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [451334144, 451338240) 0 0 00000080:00200000:1.0:1788201705.341677:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201705.341680:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [110189, 110189] 00020000:00200000:1.0:1788201705.341682:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 451334144 chunk: [451334144, 451338240) 451338240, 4096 00020000:00200000:1.0:1788201705.341683:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201705.341690:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [110189, 110189] 00000008:00010000:1.0:1788201705.341693:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201705.341694:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201705.341699:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341704:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (451334144 451338239) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 253/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341710:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341716:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 257/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341723:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 256/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.341730:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [451334144, 451338240) append 0 append lockpos 0 00000080:00200000:1.0:1788201705.341735:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 110189 of 0 to 4096 bytes 00000080:00200000:1.0:1788201705.341741:0:17463:0:(rw26.c:898:ll_write_end()) pos 451334144, len 4096, copied 4096 00000080:00200000:1.0:1788201705.341744:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000023e0b195[3 0000000075f4bf38 2 1 000000006db7d1db 0 0 110189] 00000080:00200000:1.0:1788201705.341746:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000099853193 fffffc0001029 2:0 ffff93c0c11edee0 110189 lru 00000080:00200000:1.0:1788201705.341751:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008f1943fd 110189: 1< 2 - - > 2< 451334144 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 59637760 - | + - + - > 5< + - + - | 0 - | 61774 - -> 00000080:00200000:1.0:1788201705.341754:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000023e0b195 00000080:00200000:1.0:1788201705.341755:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201705.341758:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201705.341765:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000036c0f0d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201705.341776:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201705.341778:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201705.341780:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201705.341789:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_release():245 00000020:00010000:1.0:1788201705.341791:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201705.341793:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201705.341797:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 255/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201705.341803:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 254/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201705.341811:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee8a8 (ffff93c1d1bf1488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201705.341814:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201705.341818:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201705.341821:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [110189, 110189] 00000080:00200000:1.0:1788201705.341855:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201705.341859:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.167902:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.167908:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.167934:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468303872, count: 4096 00000080:00200000:1.0:1788201706.167966:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468303872, bytes: 4096 00000080:00200000:1.0:1788201706.167969:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.167974:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.167976:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468303872, 468307968) 0 0 00000080:00200000:1.0:1788201706.167979:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.167985:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114332, 114332] 00020000:00200000:1.0:1788201706.167988:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468303872 chunk: [468303872, 468307968) 468307968, 4096 00020000:00200000:1.0:1788201706.167991:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.168017:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114332, 114332] 00000008:00010000:1.0:1788201706.168035:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.168038:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.168048:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168058:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468303872 468307967) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168068:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168076:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168083:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.168092:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468303872, 468307968) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.168098:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114332 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.168104:0:17463:0:(rw26.c:898:ll_write_end()) pos 468303872, len 4096, copied 4096 00000080:00200000:1.0:1788201706.168108:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000908913b2[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114332] 00000080:00200000:1.0:1788201706.168112:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000070b976b7 fffffc0001029 2:0 ffff93c0c12bd620 114332 lru 00000080:00200000:1.0:1788201706.168119:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a096ac12 114332: 1< 2 - - > 2< 468303872 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65917 - -> 00000080:00200000:1.0:1788201706.168122:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000908913b2 00000080:00200000:1.0:1788201706.168123:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.168125:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.168136:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.168147:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.168149:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.168151:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.168161:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.168163:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.168165:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.168167:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168173:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.168179:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.168180:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.168183:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.168186:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114332, 114332] 00000080:00200000:1.0:1788201706.168198:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.168200:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.168224:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.168228:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.168245:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468307968, count: 4096 00000080:00200000:1.0:1788201706.168256:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468307968, bytes: 4096 00000080:00200000:1.0:1788201706.168259:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.168263:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.168265:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468307968, 468312064) 0 0 00000080:00200000:1.0:1788201706.168268:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.168272:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114333, 114333] 00020000:00200000:1.0:1788201706.168275:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468307968 chunk: [468307968, 468312064) 468312064, 4096 00020000:00200000:1.0:1788201706.168276:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.168286:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114333, 114333] 00000008:00010000:1.0:1788201706.168292:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.168294:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.168302:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168311:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468307968 468312063) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168321:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168329:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168334:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.168339:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468307968, 468312064) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.168348:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114333 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.168354:0:17463:0:(rw26.c:898:ll_write_end()) pos 468307968, len 4096, copied 4096 00000080:00200000:1.0:1788201706.168359:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000f2ff3fe[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114333] 00000080:00200000:1.0:1788201706.168362:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001dc2b0dd fffffc0001029 2:0 ffff93c0c12bdd20 114333 lru 00000080:00200000:1.0:1788201706.168368:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000035e22c91 114333: 1< 2 - - > 2< 468307968 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65918 - -> 00000080:00200000:1.0:1788201706.168370:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000f2ff3fe 00000080:00200000:1.0:1788201706.168371:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.168374:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.168383:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.168396:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.168398:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.168400:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.168408:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.168409:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.168411:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.168414:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168419:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.168424:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.168426:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.168428:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.168430:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114333, 114333] 00000080:00200000:1.0:1788201706.168438:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.168440:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.168459:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.168462:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.168473:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468312064, count: 4096 00000080:00200000:1.0:1788201706.168479:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468312064, bytes: 4096 00000080:00200000:1.0:1788201706.168481:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.168483:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.168484:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468312064, 468316160) 0 0 00000080:00200000:1.0:1788201706.168491:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.168495:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114334, 114334] 00020000:00200000:1.0:1788201706.168497:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468312064 chunk: [468312064, 468316160) 468316160, 4096 00020000:00200000:1.0:1788201706.168499:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.168506:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114334, 114334] 00000008:00010000:1.0:1788201706.168509:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.168511:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.168516:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168522:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468312064 468316159) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168528:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168533:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168537:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.168541:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468312064, 468316160) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.168545:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114334 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.168549:0:17463:0:(rw26.c:898:ll_write_end()) pos 468312064, len 4096, copied 4096 00000080:00200000:1.0:1788201706.168551:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000de912498[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114334] 00000080:00200000:1.0:1788201706.168553:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ef33f423 fffffc0001029 2:0 ffff93c0c12bd9a0 114334 lru 00000080:00200000:1.0:1788201706.168557:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006816a9df 114334: 1< 2 - - > 2< 468312064 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65919 - -> 00000080:00200000:1.0:1788201706.168559:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000de912498 00000080:00200000:1.0:1788201706.168559:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.168561:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.168566:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.168574:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.168576:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.168577:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.168584:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.168585:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.168587:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.168589:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168595:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.168599:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.168601:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.168603:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.168605:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114334, 114334] 00000080:00200000:1.0:1788201706.168612:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.168614:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.168632:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.168634:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.168644:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468316160, count: 4096 00000080:00200000:1.0:1788201706.168650:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468316160, bytes: 4096 00000080:00200000:1.0:1788201706.168652:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.168654:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.168655:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468316160, 468320256) 0 0 00000080:00200000:1.0:1788201706.168657:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.168660:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114335, 114335] 00020000:00200000:1.0:1788201706.168662:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468316160 chunk: [468316160, 468320256) 468320256, 4096 00020000:00200000:1.0:1788201706.168663:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.168670:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114335, 114335] 00000008:00010000:1.0:1788201706.168673:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.168674:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.168679:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168685:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468316160 468320255) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168690:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168695:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168699:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.168707:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468316160, 468320256) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.168710:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114335 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.168714:0:17463:0:(rw26.c:898:ll_write_end()) pos 468316160, len 4096, copied 4096 00000080:00200000:1.0:1788201706.168716:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000070014122[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114335] 00000080:00200000:1.0:1788201706.168719:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000b071a1c fffffc0001029 2:0 ffff93c0c12be460 114335 lru 00000080:00200000:1.0:1788201706.168722:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000035b7ce62 114335: 1< 2 - - > 2< 468316160 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65920 - -> 00000080:00200000:1.0:1788201706.168724:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000070014122 00000080:00200000:1.0:1788201706.168724:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.168726:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.168731:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.168739:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.168741:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.168742:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.168748:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.168750:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.168751:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.168754:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168758:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.168763:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.168764:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.168768:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.168771:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114335, 114335] 00000080:00200000:1.0:1788201706.168781:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.168784:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.168810:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.168813:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.168827:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468320256, count: 4096 00000080:00200000:1.0:1788201706.168847:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468320256, bytes: 4096 00000080:00200000:1.0:1788201706.168849:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.168852:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.168853:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468320256, 468324352) 0 0 00000080:00200000:1.0:1788201706.168854:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.168858:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114336, 114336] 00020000:00200000:1.0:1788201706.168860:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468320256 chunk: [468320256, 468324352) 468324352, 4096 00020000:00200000:1.0:1788201706.168861:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.168868:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114336, 114336] 00000008:00010000:1.0:1788201706.168872:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.168873:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.168879:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168885:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468320256 468324351) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168890:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168894:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168902:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.168907:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468320256, 468324352) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.168910:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114336 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.168913:0:17463:0:(rw26.c:898:ll_write_end()) pos 468320256, len 4096, copied 4096 00000080:00200000:1.0:1788201706.168916:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b7d13f02[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114336] 00000080:00200000:1.0:1788201706.168918:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000010ae7c45 fffffc0001029 2:0 ffff93c0c12be0e0 114336 lru 00000080:00200000:1.0:1788201706.168921:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008e24de2d 114336: 1< 2 - - > 2< 468320256 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65921 - -> 00000080:00200000:1.0:1788201706.168923:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b7d13f02 00000080:00200000:1.0:1788201706.168924:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.168925:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.168933:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.168944:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.168947:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.168949:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.168958:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.168961:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.168963:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.168967:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.168975:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.168988:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.168990:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.168994:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.168997:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114336, 114336] 00000080:00200000:1.0:1788201706.169008:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.169011:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.169056:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.169060:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.169078:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468324352, count: 4096 00000080:00200000:1.0:1788201706.169089:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468324352, bytes: 4096 00000080:00200000:1.0:1788201706.169092:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.169096:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.169097:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468324352, 468328448) 0 0 00000080:00200000:1.0:1788201706.169100:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.169105:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114337, 114337] 00020000:00200000:1.0:1788201706.169108:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468324352 chunk: [468324352, 468328448) 468328448, 4096 00020000:00200000:1.0:1788201706.169111:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.169123:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114337, 114337] 00000008:00010000:1.0:1788201706.169128:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.169130:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.169138:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169147:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468324352 468328447) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169153:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169162:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169166:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.169171:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468324352, 468328448) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.169174:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114337 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.169178:0:17463:0:(rw26.c:898:ll_write_end()) pos 468324352, len 4096, copied 4096 00000080:00200000:1.0:1788201706.169181:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000029f86585[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114337] 00000080:00200000:1.0:1788201706.169182:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f8e073d6 fffffc0001029 2:0 ffff93c0c12be700 114337 lru 00000080:00200000:1.0:1788201706.169186:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006563c4a2 114337: 1< 2 - - > 2< 468324352 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65922 - -> 00000080:00200000:1.0:1788201706.169187:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000029f86585 00000080:00200000:1.0:1788201706.169188:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.169189:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.169195:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.169204:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.169206:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.169208:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.169217:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.169219:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.169222:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.169226:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169239:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.169247:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.169250:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.169254:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.169257:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114337, 114337] 00000080:00200000:1.0:1788201706.169266:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.169268:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.169289:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.169293:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.169308:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468328448, count: 4096 00000080:00200000:1.0:1788201706.169319:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468328448, bytes: 4096 00000080:00200000:1.0:1788201706.169322:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.169326:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.169328:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468328448, 468332544) 0 0 00000080:00200000:1.0:1788201706.169330:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.169335:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114338, 114338] 00020000:00200000:1.0:1788201706.169338:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468328448 chunk: [468328448, 468332544) 468332544, 4096 00020000:00200000:1.0:1788201706.169339:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.169347:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114338, 114338] 00000008:00010000:1.0:1788201706.169351:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.169353:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.169361:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169369:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468328448 468332543) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169384:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169392:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169398:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.169404:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468328448, 468332544) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.169408:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114338 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.169411:0:17463:0:(rw26.c:898:ll_write_end()) pos 468328448, len 4096, copied 4096 00000080:00200000:1.0:1788201706.169414:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000444f3a9a[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114338] 00000080:00200000:1.0:1788201706.169416:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004f3d2f59 fffffc0001029 2:0 ffff93c0c12be000 114338 lru 00000080:00200000:1.0:1788201706.169420:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c46435ae 114338: 1< 2 - - > 2< 468328448 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65923 - -> 00000080:00200000:1.0:1788201706.169421:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000444f3a9a 00000080:00200000:1.0:1788201706.169422:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.169423:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.169429:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.169438:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.169439:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.169441:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.169447:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.169449:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.169451:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.169454:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169459:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.169464:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.169465:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.169468:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.169470:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114338, 114338] 00000080:00200000:1.0:1788201706.169477:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.169479:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.169496:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.169499:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.169509:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468332544, count: 4096 00000080:00200000:1.0:1788201706.169515:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468332544, bytes: 4096 00000080:00200000:1.0:1788201706.169517:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.169519:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.169520:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468332544, 468336640) 0 0 00000080:00200000:1.0:1788201706.169522:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.169525:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114339, 114339] 00020000:00200000:1.0:1788201706.169527:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468332544 chunk: [468332544, 468336640) 468336640, 4096 00020000:00200000:1.0:1788201706.169528:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.169535:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114339, 114339] 00000008:00010000:1.0:1788201706.169538:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.169540:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.169545:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169554:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468332544 468336639) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169559:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169563:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169567:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.169572:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468332544, 468336640) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.169575:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114339 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.169578:0:17463:0:(rw26.c:898:ll_write_end()) pos 468332544, len 4096, copied 4096 00000080:00200000:1.0:1788201706.169581:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000097f2cb2e[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114339] 00000080:00200000:1.0:1788201706.169583:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b0a63939 fffffc0001029 2:0 ffff93c0c12bea80 114339 lru 00000080:00200000:1.0:1788201706.169586:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000091d0ae64 114339: 1< 2 - - > 2< 468332544 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65924 - -> 00000080:00200000:1.0:1788201706.169588:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000097f2cb2e 00000080:00200000:1.0:1788201706.169588:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.169590:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.169595:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.169603:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.169604:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.169606:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.169612:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.169614:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.169615:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.169617:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169622:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.169626:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.169628:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.169630:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.169632:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114339, 114339] 00000080:00200000:1.0:1788201706.169639:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.169641:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.169658:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.169660:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.169670:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468336640, count: 4096 00000080:00200000:1.0:1788201706.169676:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468336640, bytes: 4096 00000080:00200000:1.0:1788201706.169677:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.169680:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.169681:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468336640, 468340736) 0 0 00000080:00200000:1.0:1788201706.169682:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.169685:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114340, 114340] 00020000:00200000:1.0:1788201706.169687:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468336640 chunk: [468336640, 468340736) 468340736, 4096 00020000:00200000:1.0:1788201706.169689:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.169695:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114340, 114340] 00000008:00010000:1.0:1788201706.169698:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.169700:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.169704:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169710:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468336640 468340735) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169715:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169720:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169723:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.169727:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468336640, 468340736) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.169730:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114340 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.169733:0:17463:0:(rw26.c:898:ll_write_end()) pos 468336640, len 4096, copied 4096 00000080:00200000:1.0:1788201706.169736:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005105868f[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114340] 00000080:00200000:1.0:1788201706.169738:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000733458da fffffc0001029 2:0 ffff93c0c12be380 114340 lru 00000080:00200000:1.0:1788201706.169741:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000096eb977b 114340: 1< 2 - - > 2< 468336640 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65925 - -> 00000080:00200000:1.0:1788201706.169743:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005105868f 00000080:00200000:1.0:1788201706.169743:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.169745:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.169750:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.169757:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.169759:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.169760:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.169766:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.169769:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.169771:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.169775:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169783:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.169790:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.169793:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.169797:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.169800:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114340, 114340] 00000080:00200000:1.0:1788201706.169809:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.169812:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.169842:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.169846:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.169861:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468340736, count: 4096 00000080:00200000:1.0:1788201706.169872:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468340736, bytes: 4096 00000080:00200000:1.0:1788201706.169874:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.169878:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.169880:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468340736, 468344832) 0 0 00000080:00200000:1.0:1788201706.169882:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.169887:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114341, 114341] 00020000:00200000:1.0:1788201706.169890:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468340736 chunk: [468340736, 468344832) 468344832, 4096 00020000:00200000:1.0:1788201706.169892:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.169904:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114341, 114341] 00000008:00010000:1.0:1788201706.169909:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.169912:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.169919:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169928:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468340736 468344831) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169937:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169945:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.169953:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.169961:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468340736, 468344832) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.169966:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114341 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.169972:0:17463:0:(rw26.c:898:ll_write_end()) pos 468340736, len 4096, copied 4096 00000080:00200000:1.0:1788201706.169976:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ac1c6a03[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114341] 00000080:00200000:1.0:1788201706.169979:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000050188b78 fffffc0001029 2:0 ffff93c0c12be8c0 114341 lru 00000080:00200000:1.0:1788201706.169985:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000034a931f7 114341: 1< 2 - - > 2< 468340736 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65926 - -> 00000080:00200000:1.0:1788201706.169987:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ac1c6a03 00000080:00200000:1.0:1788201706.169988:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.169990:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.169999:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.170017:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.170019:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.170022:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.170046:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.170049:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.170052:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.170056:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170065:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.170073:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.170075:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.170078:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.170081:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114341, 114341] 00000080:00200000:1.0:1788201706.170088:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.170090:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.170110:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.170112:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.170123:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468344832, count: 4096 00000080:00200000:1.0:1788201706.170129:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468344832, bytes: 4096 00000080:00200000:1.0:1788201706.170131:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.170133:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.170134:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468344832, 468348928) 0 0 00000080:00200000:1.0:1788201706.170136:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.170139:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114342, 114342] 00020000:00200000:1.0:1788201706.170141:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468344832 chunk: [468344832, 468348928) 468348928, 4096 00020000:00200000:1.0:1788201706.170143:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.170149:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114342, 114342] 00000008:00010000:1.0:1788201706.170153:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.170154:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.170159:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170165:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468344832 468348927) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170171:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170175:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170179:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.170184:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468344832, 468348928) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.170187:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114342 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.170191:0:17463:0:(rw26.c:898:ll_write_end()) pos 468344832, len 4096, copied 4096 00000080:00200000:1.0:1788201706.170193:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a8487034[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114342] 00000080:00200000:1.0:1788201706.170195:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000612e5c7 fffffc0001029 2:0 ffff93c0c12beee0 114342 lru 00000080:00200000:1.0:1788201706.170199:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002560bab3 114342: 1< 2 - - > 2< 468344832 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65927 - -> 00000080:00200000:1.0:1788201706.170201:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a8487034 00000080:00200000:1.0:1788201706.170202:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.170204:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.170212:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.170223:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.170226:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.170228:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.170237:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.170240:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.170242:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.170246:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170254:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.170260:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.170263:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.170267:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.170270:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114342, 114342] 00000080:00200000:1.0:1788201706.170281:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.170284:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.170312:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.170315:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.170326:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468348928, count: 4096 00000080:00200000:1.0:1788201706.170333:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468348928, bytes: 4096 00000080:00200000:1.0:1788201706.170334:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.170337:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.170338:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468348928, 468353024) 0 0 00000080:00200000:1.0:1788201706.170340:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.170343:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114343, 114343] 00020000:00200000:1.0:1788201706.170346:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468348928 chunk: [468348928, 468353024) 468353024, 4096 00020000:00200000:1.0:1788201706.170347:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.170357:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114343, 114343] 00000008:00010000:1.0:1788201706.170362:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.170364:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.170371:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170380:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468348928 468353023) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170389:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170398:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170403:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.170408:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468348928, 468353024) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.170411:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114343 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.170415:0:17463:0:(rw26.c:898:ll_write_end()) pos 468348928, len 4096, copied 4096 00000080:00200000:1.0:1788201706.170417:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c74bf91c[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114343] 00000080:00200000:1.0:1788201706.170419:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006f5ef576 fffffc0001029 2:0 ffff93c0c12be7e0 114343 lru 00000080:00200000:1.0:1788201706.170423:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008b04d42d 114343: 1< 2 - - > 2< 468348928 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65928 - -> 00000080:00200000:1.0:1788201706.170425:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c74bf91c 00000080:00200000:1.0:1788201706.170426:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.170427:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.170433:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.170442:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.170443:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.170445:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.170451:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.170453:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.170454:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.170457:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170462:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.170466:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.170468:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.170470:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.170472:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114343, 114343] 00000080:00200000:1.0:1788201706.170479:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.170481:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.170499:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.170501:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.170511:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468353024, count: 4096 00000080:00200000:1.0:1788201706.170517:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468353024, bytes: 4096 00000080:00200000:1.0:1788201706.170519:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.170521:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.170523:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468353024, 468357120) 0 0 00000080:00200000:1.0:1788201706.170524:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.170527:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114344, 114344] 00020000:00200000:1.0:1788201706.170529:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468353024 chunk: [468353024, 468357120) 468357120, 4096 00020000:00200000:1.0:1788201706.170531:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.170537:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114344, 114344] 00000008:00010000:1.0:1788201706.170540:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.170542:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.170547:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170553:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468353024 468357119) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170558:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170563:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170567:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.170571:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468353024, 468357120) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.170573:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114344 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.170577:0:17463:0:(rw26.c:898:ll_write_end()) pos 468353024, len 4096, copied 4096 00000080:00200000:1.0:1788201706.170579:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000464e5eca[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114344] 00000080:00200000:1.0:1788201706.170582:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ae12b237 fffffc0001029 2:0 ffff93c0c12be2a0 114344 lru 00000080:00200000:1.0:1788201706.170585:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bc5e3e52 114344: 1< 2 - - > 2< 468353024 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65929 - -> 00000080:00200000:1.0:1788201706.170587:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000464e5eca 00000080:00200000:1.0:1788201706.170587:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.170589:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.170594:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.170601:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.170602:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.170604:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.170610:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.170611:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.170613:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.170615:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170620:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.170625:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.170626:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.170628:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.170630:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114344, 114344] 00000080:00200000:1.0:1788201706.170636:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.170638:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.170655:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.170657:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.170667:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468357120, count: 4096 00000080:00200000:1.0:1788201706.170673:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468357120, bytes: 4096 00000080:00200000:1.0:1788201706.170675:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.170677:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.170678:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468357120, 468361216) 0 0 00000080:00200000:1.0:1788201706.170680:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.170683:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114345, 114345] 00020000:00200000:1.0:1788201706.170684:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468357120 chunk: [468357120, 468361216) 468361216, 4096 00020000:00200000:1.0:1788201706.170686:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.170692:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114345, 114345] 00000008:00010000:1.0:1788201706.170695:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.170696:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.170701:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170706:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468357120 468361215) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170712:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170716:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170720:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.170725:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468357120, 468361216) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.170728:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114345 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.170732:0:17463:0:(rw26.c:898:ll_write_end()) pos 468357120, len 4096, copied 4096 00000080:00200000:1.0:1788201706.170734:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000053f47abc[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114345] 00000080:00200000:1.0:1788201706.170736:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000008d98471 fffffc0001029 2:0 ffff93c0c12bee00 114345 lru 00000080:00200000:1.0:1788201706.170740:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f0cd4942 114345: 1< 2 - - > 2< 468357120 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65930 - -> 00000080:00200000:1.0:1788201706.170743:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000053f47abc 00000080:00200000:1.0:1788201706.170744:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.170746:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.170753:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.170765:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.170768:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.170769:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.170778:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.170781:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.170783:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.170787:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170795:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.170804:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.170805:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.170808:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.170810:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114345, 114345] 00000080:00200000:1.0:1788201706.170817:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.170819:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.170852:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.170854:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.170864:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468361216, count: 4096 00000080:00200000:1.0:1788201706.170871:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468361216, bytes: 4096 00000080:00200000:1.0:1788201706.170872:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.170875:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.170876:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468361216, 468365312) 0 0 00000080:00200000:1.0:1788201706.170877:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.170880:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114346, 114346] 00020000:00200000:1.0:1788201706.170882:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468361216 chunk: [468361216, 468365312) 468365312, 4096 00020000:00200000:1.0:1788201706.170884:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.170890:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114346, 114346] 00000008:00010000:1.0:1788201706.170894:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.170895:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.170900:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170906:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468361216 468365311) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170912:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170917:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170921:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.170925:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468361216, 468365312) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.170928:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114346 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.170932:0:17463:0:(rw26.c:898:ll_write_end()) pos 468361216, len 4096, copied 4096 00000080:00200000:1.0:1788201706.170934:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c7765989[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114346] 00000080:00200000:1.0:1788201706.170936:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e3198a96 fffffc0001029 2:0 ffff93c0c12be1c0 114346 lru 00000080:00200000:1.0:1788201706.170940:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000ee0db51 114346: 1< 2 - - > 2< 468361216 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65931 - -> 00000080:00200000:1.0:1788201706.170941:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c7765989 00000080:00200000:1.0:1788201706.170942:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.170943:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.170948:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.170956:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.170957:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.170959:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.170965:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.170966:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.170968:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.170970:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.170975:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.170980:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.170981:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.170984:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.170986:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114346, 114346] 00000080:00200000:1.0:1788201706.170992:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.170994:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.171011:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.171013:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.171035:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468365312, count: 4096 00000080:00200000:1.0:1788201706.171044:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468365312, bytes: 4096 00000080:00200000:1.0:1788201706.171045:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.171048:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.171049:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468365312, 468369408) 0 0 00000080:00200000:1.0:1788201706.171050:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.171054:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114347, 114347] 00020000:00200000:1.0:1788201706.171056:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468365312 chunk: [468365312, 468369408) 468369408, 4096 00020000:00200000:1.0:1788201706.171057:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.171064:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114347, 114347] 00000008:00010000:1.0:1788201706.171067:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.171068:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.171073:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171079:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468365312 468369407) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171084:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171089:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171093:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.171100:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468365312, 468369408) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.171103:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114347 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.171107:0:17463:0:(rw26.c:898:ll_write_end()) pos 468365312, len 4096, copied 4096 00000080:00200000:1.0:1788201706.171109:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000cac9d04b[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114347] 00000080:00200000:1.0:1788201706.171111:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d39585b6 fffffc0001029 2:0 ffff93c0c12be620 114347 lru 00000080:00200000:1.0:1788201706.171115:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e1491376 114347: 1< 2 - - > 2< 468365312 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65932 - -> 00000080:00200000:1.0:1788201706.171118:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000cac9d04b 00000080:00200000:1.0:1788201706.171119:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.171121:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.171128:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.171139:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.171142:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.171144:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.171153:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.171155:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.171158:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.171162:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171169:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.171175:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.171176:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.171179:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.171181:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114347, 114347] 00000080:00200000:1.0:1788201706.171189:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.171191:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.171214:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.171217:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.171232:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468369408, count: 4096 00000080:00200000:1.0:1788201706.171243:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468369408, bytes: 4096 00000080:00200000:1.0:1788201706.171246:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.171250:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.171251:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468369408, 468373504) 0 0 00000080:00200000:1.0:1788201706.171253:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.171257:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114348, 114348] 00020000:00200000:1.0:1788201706.171261:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468369408 chunk: [468369408, 468373504) 468373504, 4096 00020000:00200000:1.0:1788201706.171263:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.171275:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114348, 114348] 00000008:00010000:1.0:1788201706.171280:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.171282:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.171289:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171298:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468369408 468373503) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171305:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171310:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171318:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.171323:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468369408, 468373504) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.171326:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114348 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.171329:0:17463:0:(rw26.c:898:ll_write_end()) pos 468369408, len 4096, copied 4096 00000080:00200000:1.0:1788201706.171332:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000900212d7[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114348] 00000080:00200000:1.0:1788201706.171333:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fea9395b fffffc0001029 2:0 ffff93c0c12bed20 114348 lru 00000080:00200000:1.0:1788201706.171337:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e0757b40 114348: 1< 2 - - > 2< 468369408 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65933 - -> 00000080:00200000:1.0:1788201706.171338:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000900212d7 00000080:00200000:1.0:1788201706.171339:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.171340:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.171346:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.171356:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.171359:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.171361:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.171370:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.171372:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.171374:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.171378:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171387:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.171400:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.171402:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.171405:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.171407:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114348, 114348] 00000080:00200000:1.0:1788201706.171414:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.171416:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.171435:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.171437:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.171447:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468373504, count: 4096 00000080:00200000:1.0:1788201706.171453:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468373504, bytes: 4096 00000080:00200000:1.0:1788201706.171455:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.171457:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.171458:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468373504, 468377600) 0 0 00000080:00200000:1.0:1788201706.171459:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.171463:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114349, 114349] 00020000:00200000:1.0:1788201706.171465:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468373504 chunk: [468373504, 468377600) 468377600, 4096 00020000:00200000:1.0:1788201706.171466:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.171472:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114349, 114349] 00000008:00010000:1.0:1788201706.171476:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.171477:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.171482:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171488:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468373504 468377599) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171493:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171502:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171506:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.171510:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468373504, 468377600) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.171512:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114349 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.171516:0:17463:0:(rw26.c:898:ll_write_end()) pos 468373504, len 4096, copied 4096 00000080:00200000:1.0:1788201706.171518:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fa7fe7f2[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114349] 00000080:00200000:1.0:1788201706.171520:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000997eaf04 fffffc0001029 2:0 ffff93c0c12be9a0 114349 lru 00000080:00200000:1.0:1788201706.171523:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cba9dc3b 114349: 1< 2 - - > 2< 468373504 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65934 - -> 00000080:00200000:1.0:1788201706.171524:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fa7fe7f2 00000080:00200000:1.0:1788201706.171525:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.171526:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.171532:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.171539:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.171541:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.171542:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.171548:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.171550:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.171551:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.171554:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171562:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.171566:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.171568:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.171570:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.171572:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114349, 114349] 00000080:00200000:1.0:1788201706.171578:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.171580:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.171597:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.171599:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.171609:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468377600, count: 4096 00000080:00200000:1.0:1788201706.171615:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468377600, bytes: 4096 00000080:00200000:1.0:1788201706.171617:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.171619:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.171620:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468377600, 468381696) 0 0 00000080:00200000:1.0:1788201706.171621:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.171624:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114350, 114350] 00020000:00200000:1.0:1788201706.171626:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468377600 chunk: [468377600, 468381696) 468381696, 4096 00020000:00200000:1.0:1788201706.171628:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.171634:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114350, 114350] 00000008:00010000:1.0:1788201706.171637:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.171638:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.171643:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171648:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468377600 468381695) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171657:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171661:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171665:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.171670:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468377600, 468381696) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.171672:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114350 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.171676:0:17463:0:(rw26.c:898:ll_write_end()) pos 468377600, len 4096, copied 4096 00000080:00200000:1.0:1788201706.171678:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003d45f6a3[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114350] 00000080:00200000:1.0:1788201706.171680:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007b6ccda9 fffffc0001029 2:0 ffff93c0c12beb60 114350 lru 00000080:00200000:1.0:1788201706.171684:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001e53e9c7 114350: 1< 2 - - > 2< 468377600 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65935 - -> 00000080:00200000:1.0:1788201706.171685:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003d45f6a3 00000080:00200000:1.0:1788201706.171686:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.171687:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.171692:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.171700:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.171701:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.171703:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.171708:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.171710:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.171712:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.171714:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171719:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.171724:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.171726:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.171728:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.171730:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114350, 114350] 00000080:00200000:1.0:1788201706.171737:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.171740:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.171766:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.171770:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.171785:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468381696, count: 4096 00000080:00200000:1.0:1788201706.171794:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468381696, bytes: 4096 00000080:00200000:1.0:1788201706.171795:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.171798:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.171799:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468381696, 468385792) 0 0 00000080:00200000:1.0:1788201706.171800:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.171804:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114351, 114351] 00020000:00200000:1.0:1788201706.171806:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468381696 chunk: [468381696, 468385792) 468385792, 4096 00020000:00200000:1.0:1788201706.171808:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.171814:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114351, 114351] 00000008:00010000:1.0:1788201706.171818:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.171819:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.171824:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171846:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468381696 468385791) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171852:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171857:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171861:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.171865:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468381696, 468385792) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.171868:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114351 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.171872:0:17463:0:(rw26.c:898:ll_write_end()) pos 468381696, len 4096, copied 4096 00000080:00200000:1.0:1788201706.171874:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000008690648[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114351] 00000080:00200000:1.0:1788201706.171876:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000071ff474a fffffc0001029 2:0 ffff93c0c12bec40 114351 lru 00000080:00200000:1.0:1788201706.171879:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000042f1ab7c 114351: 1< 2 - - > 2< 468381696 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65936 - -> 00000080:00200000:1.0:1788201706.171881:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000008690648 00000080:00200000:1.0:1788201706.171881:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.171883:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.171888:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.171896:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.171898:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.171899:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.171905:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.171907:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.171908:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.171911:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.171916:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.171921:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.171922:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.171924:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.171926:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114351, 114351] 00000080:00200000:1.0:1788201706.171933:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.171934:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.171952:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.171955:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.171964:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468385792, count: 4096 00000080:00200000:1.0:1788201706.171970:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468385792, bytes: 4096 00000080:00200000:1.0:1788201706.171972:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.171974:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.171975:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468385792, 468389888) 0 0 00000080:00200000:1.0:1788201706.171977:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.171980:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114352, 114352] 00020000:00200000:1.0:1788201706.171982:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468385792 chunk: [468385792, 468389888) 468389888, 4096 00020000:00200000:1.0:1788201706.171983:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.171989:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114352, 114352] 00000008:00010000:1.0:1788201706.171992:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.171994:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.171999:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172004:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468385792 468389887) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172010:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172014:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172018:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.172034:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468385792, 468389888) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.172038:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114352 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.172042:0:17463:0:(rw26.c:898:ll_write_end()) pos 468385792, len 4096, copied 4096 00000080:00200000:1.0:1788201706.172045:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002ad86ba5[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114352] 00000080:00200000:1.0:1788201706.172047:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006149b33a fffffc0001029 2:0 ffff93c0c12be540 114352 lru 00000080:00200000:1.0:1788201706.172050:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000dad27970 114352: 1< 2 - - > 2< 468385792 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65937 - -> 00000080:00200000:1.0:1788201706.172052:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002ad86ba5 00000080:00200000:1.0:1788201706.172053:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.172054:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.172060:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.172067:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.172069:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.172071:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.172077:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.172079:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.172080:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.172083:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172088:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.172093:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.172094:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.172096:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.172098:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114352, 114352] 00000080:00200000:1.0:1788201706.172105:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.172107:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.172131:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.172134:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.172150:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468389888, count: 4096 00000080:00200000:1.0:1788201706.172162:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468389888, bytes: 4096 00000080:00200000:1.0:1788201706.172164:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.172166:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.172167:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468389888, 468393984) 0 0 00000080:00200000:1.0:1788201706.172169:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.172172:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114353, 114353] 00020000:00200000:1.0:1788201706.172175:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468389888 chunk: [468389888, 468393984) 468393984, 4096 00020000:00200000:1.0:1788201706.172176:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.172183:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114353, 114353] 00000008:00010000:1.0:1788201706.172187:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.172188:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.172193:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172199:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468389888 468393983) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172207:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172214:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172221:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.172229:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468389888, 468393984) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.172234:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114353 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.172239:0:17463:0:(rw26.c:898:ll_write_end()) pos 468389888, len 4096, copied 4096 00000080:00200000:1.0:1788201706.172243:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008e7c1ba5[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114353] 00000080:00200000:1.0:1788201706.172246:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008784fa79 fffffc0001029 2:0 ffff93c0c12bf620 114353 lru 00000080:00200000:1.0:1788201706.172250:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009df0cf6b 114353: 1< 2 - - > 2< 468389888 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65938 - -> 00000080:00200000:1.0:1788201706.172252:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008e7c1ba5 00000080:00200000:1.0:1788201706.172253:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.172254:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.172263:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.172280:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.172283:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.172285:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.172295:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.172297:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.172300:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.172302:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172308:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.172313:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.172315:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.172317:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.172320:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114353, 114353] 00000080:00200000:1.0:1788201706.172327:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.172329:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.172347:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.172350:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.172364:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468393984, count: 4096 00000080:00200000:1.0:1788201706.172373:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468393984, bytes: 4096 00000080:00200000:1.0:1788201706.172376:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.172380:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.172382:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468393984, 468398080) 0 0 00000080:00200000:1.0:1788201706.172384:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.172389:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114354, 114354] 00020000:00200000:1.0:1788201706.172393:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468393984 chunk: [468393984, 468398080) 468398080, 4096 00020000:00200000:1.0:1788201706.172395:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.172405:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114354, 114354] 00000008:00010000:1.0:1788201706.172408:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.172410:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.172415:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172421:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468393984 468398079) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172426:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172431:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172435:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.172440:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468393984, 468398080) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.172443:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114354 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.172447:0:17463:0:(rw26.c:898:ll_write_end()) pos 468393984, len 4096, copied 4096 00000080:00200000:1.0:1788201706.172449:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004888d975[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114354] 00000080:00200000:1.0:1788201706.172451:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fead0062 fffffc0001029 2:0 ffff93c0c12bfd20 114354 lru 00000080:00200000:1.0:1788201706.172455:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005dee10e9 114354: 1< 2 - - > 2< 468393984 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65939 - -> 00000080:00200000:1.0:1788201706.172456:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004888d975 00000080:00200000:1.0:1788201706.172457:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.172458:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.172464:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.172472:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.172473:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.172475:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.172481:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.172482:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.172484:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.172486:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172492:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.172496:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.172497:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.172500:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.172502:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114354, 114354] 00000080:00200000:1.0:1788201706.172508:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.172510:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.172528:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.172530:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.172540:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468398080, count: 4096 00000080:00200000:1.0:1788201706.172546:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468398080, bytes: 4096 00000080:00200000:1.0:1788201706.172548:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.172550:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.172551:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468398080, 468402176) 0 0 00000080:00200000:1.0:1788201706.172552:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.172556:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114355, 114355] 00020000:00200000:1.0:1788201706.172558:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468398080 chunk: [468398080, 468402176) 468402176, 4096 00020000:00200000:1.0:1788201706.172559:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.172565:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114355, 114355] 00000008:00010000:1.0:1788201706.172568:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.172570:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.172574:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172580:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468398080 468402175) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172585:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172590:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172594:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.172598:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468398080, 468402176) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.172600:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114355 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.172604:0:17463:0:(rw26.c:898:ll_write_end()) pos 468398080, len 4096, copied 4096 00000080:00200000:1.0:1788201706.172606:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000075a25da3[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114355] 00000080:00200000:1.0:1788201706.172607:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000010ddd858 fffffc0001029 2:0 ffff93c0c12bf9a0 114355 lru 00000080:00200000:1.0:1788201706.172611:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000834cad04 114355: 1< 2 - - > 2< 468398080 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65940 - -> 00000080:00200000:1.0:1788201706.172614:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000075a25da3 00000080:00200000:1.0:1788201706.172614:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.172616:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.172621:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.172628:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.172630:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.172631:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.172637:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.172638:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.172640:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.172642:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172646:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.172651:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.172652:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.172655:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.172657:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114355, 114355] 00000080:00200000:1.0:1788201706.172663:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.172665:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.172682:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.172684:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.172694:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 468402176, count: 4096 00000080:00200000:1.0:1788201706.172700:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 468402176, bytes: 4096 00000080:00200000:1.0:1788201706.172702:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.172704:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.172705:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [468402176, 468406272) 0 0 00000080:00200000:1.0:1788201706.172707:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.172710:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [114356, 114356] 00020000:00200000:1.0:1788201706.172711:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 468402176 chunk: [468402176, 468406272) 468406272, 4096 00020000:00200000:1.0:1788201706.172713:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.172719:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [114356, 114356] 00000008:00010000:1.0:1788201706.172722:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.172723:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.172728:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172734:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (468402176 468406271) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 269/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172741:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172749:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 273/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172756:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 272/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.172764:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [468402176, 468406272) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.172769:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 114356 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.172773:0:17463:0:(rw26.c:898:ll_write_end()) pos 468402176, len 4096, copied 4096 00000080:00200000:1.0:1788201706.172778:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000069a8ee6[3 0000000075f4bf38 2 1 00000000bca47805 0 0 114356] 00000080:00200000:1.0:1788201706.172781:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ffc08de4 fffffc0001029 2:0 ffff93c0c12bfb60 114356 lru 00000080:00200000:1.0:1788201706.172786:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000070fddf7c 114356: 1< 2 - - > 2< 468402176 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 32374784 - | + - + - > 5< + - + - | 0 - | 65941 - -> 00000080:00200000:1.0:1788201706.172788:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000069a8ee6 00000080:00200000:1.0:1788201706.172788:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.172790:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.172796:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000005cee2e10 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.172804:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.172806:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.172807:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.172813:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.172815:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.172816:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.172819:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 271/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.172824:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 270/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.172847:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee998 (ffff93c1d1bf1248/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.172849:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf1248/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.172852:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.172854:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [114356, 114356] 00000080:00200000:1.0:1788201706.172861:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.172863:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.600981:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.600984:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.600986:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.600996:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.600998:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.601001:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.601005:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601013:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.601036:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.601039:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.601044:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.601047:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116522, 116522] 00000080:00200000:0.0:1788201706.601058:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.601062:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.601092:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.601096:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.601112:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477278208, count: 4096 00000080:00200000:0.0:1788201706.601123:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477278208, bytes: 4096 00000080:00200000:0.0:1788201706.601126:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.601130:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.601131:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477278208, 477282304) 0 0 00000080:00200000:0.0:1788201706.601134:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.601139:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116523, 116523] 00020000:00200000:0.0:1788201706.601142:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477278208 chunk: [477278208, 477282304) 477282304, 4096 00020000:00200000:0.0:1788201706.601144:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.601156:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116523, 116523] 00000008:00010000:0.0:1788201706.601162:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.601164:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.601172:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601181:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477278208 477282303) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601189:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601197:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601204:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.601211:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477278208, 477282304) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.601216:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116523 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.601222:0:17463:0:(rw26.c:898:ll_write_end()) pos 477278208, len 4096, copied 4096 00000080:00200000:0.0:1788201706.601226:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d9071ee1[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116523] 00000080:00200000:0.0:1788201706.601229:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005f0e005b fffffc0001029 2:0 ffff93c0c12d7620 116523 lru 00000080:00200000:0.0:1788201706.601234:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f261957c 116523: 1< 2 - - > 2< 477278208 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68108 - -> 00000080:00200000:0.0:1788201706.601237:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d9071ee1 00000080:00200000:0.0:1788201706.601238:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.601239:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.601248:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.601264:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.601267:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.601269:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.601279:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.601282:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.601284:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.601288:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601296:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.601301:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.601303:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.601306:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.601308:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116523, 116523] 00000080:00200000:0.0:1788201706.601316:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.601318:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.601337:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.601340:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.601350:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477282304, count: 4096 00000080:00200000:0.0:1788201706.601356:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477282304, bytes: 4096 00000080:00200000:0.0:1788201706.601358:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.601360:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.601362:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477282304, 477286400) 0 0 00000080:00200000:0.0:1788201706.601363:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.601367:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116524, 116524] 00020000:00200000:0.0:1788201706.601369:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477282304 chunk: [477282304, 477286400) 477286400, 4096 00020000:00200000:0.0:1788201706.601370:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.601377:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116524, 116524] 00000008:00010000:0.0:1788201706.601380:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.601382:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.601387:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601393:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477282304 477286399) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601398:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601403:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601406:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.601411:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477282304, 477286400) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.601413:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116524 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.601417:0:17463:0:(rw26.c:898:ll_write_end()) pos 477282304, len 4096, copied 4096 00000080:00200000:0.0:1788201706.601419:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000270a93be[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116524] 00000080:00200000:0.0:1788201706.601421:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a5a040a2 fffffc0001029 2:0 ffff93c0c12d7d20 116524 lru 00000080:00200000:0.0:1788201706.601424:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000021aa5853 116524: 1< 2 - - > 2< 477282304 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68109 - -> 00000080:00200000:0.0:1788201706.601425:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000270a93be 00000080:00200000:0.0:1788201706.601426:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.601428:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.601433:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.601441:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.601443:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.601444:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.601450:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.601451:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.601453:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.601455:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601460:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.601464:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.601466:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.601468:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.601470:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116524, 116524] 00000080:00200000:0.0:1788201706.601476:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.601478:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.601502:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.601506:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.601522:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477286400, count: 4096 00000080:00200000:0.0:1788201706.601532:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477286400, bytes: 4096 00000080:00200000:0.0:1788201706.601535:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.601539:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.601540:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477286400, 477290496) 0 0 00000080:00200000:0.0:1788201706.601544:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.601548:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116525, 116525] 00020000:00200000:0.0:1788201706.601550:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477286400 chunk: [477286400, 477290496) 477290496, 4096 00020000:00200000:0.0:1788201706.601551:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.601559:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116525, 116525] 00000008:00010000:0.0:1788201706.601563:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.601564:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.601570:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601575:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477286400 477290495) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601581:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601586:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601590:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.601594:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477286400, 477290496) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.601597:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116525 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.601601:0:17463:0:(rw26.c:898:ll_write_end()) pos 477286400, len 4096, copied 4096 00000080:00200000:0.0:1788201706.601604:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b8eb73dc[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116525] 00000080:00200000:0.0:1788201706.601606:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009eb65abb fffffc0001029 2:0 ffff93c0c12d79a0 116525 lru 00000080:00200000:0.0:1788201706.601609:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000040897de9 116525: 1< 2 - - > 2< 477286400 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68110 - -> 00000080:00200000:0.0:1788201706.601612:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b8eb73dc 00000080:00200000:0.0:1788201706.601612:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.601614:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.601619:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.601627:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.601629:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.601630:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.601636:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.601638:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.601639:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.601642:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601646:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.601651:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.601652:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.601655:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.601657:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116525, 116525] 00000080:00200000:0.0:1788201706.601663:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.601665:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.601683:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.601686:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.601696:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477290496, count: 4096 00000080:00200000:0.0:1788201706.601702:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477290496, bytes: 4096 00000080:00200000:0.0:1788201706.601704:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.601706:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.601707:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477290496, 477294592) 0 0 00000080:00200000:0.0:1788201706.601709:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.601712:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116526, 116526] 00020000:00200000:0.0:1788201706.601713:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477290496 chunk: [477290496, 477294592) 477294592, 4096 00020000:00200000:0.0:1788201706.601715:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.601721:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116526, 116526] 00000008:00010000:0.0:1788201706.601724:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.601725:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.601730:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601736:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477290496 477294591) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601741:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601745:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601749:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.601754:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477290496, 477294592) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.601757:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116526 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.601760:0:17463:0:(rw26.c:898:ll_write_end()) pos 477290496, len 4096, copied 4096 00000080:00200000:0.0:1788201706.601763:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003677a31f[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116526] 00000080:00200000:0.0:1788201706.601765:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007838ac1d fffffc0001029 2:0 ffff93c0c12d7b60 116526 lru 00000080:00200000:0.0:1788201706.601768:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003979b87e 116526: 1< 2 - - > 2< 477290496 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68111 - -> 00000080:00200000:0.0:1788201706.601770:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003677a31f 00000080:00200000:0.0:1788201706.601770:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.601772:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.601777:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.601784:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.601786:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.601787:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.601793:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.601794:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.601796:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.601798:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601803:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.601807:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.601809:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.601811:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.601813:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116526, 116526] 00000080:00200000:0.0:1788201706.601819:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.601821:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.601863:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.601866:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.601879:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477294592, count: 4096 00000080:00200000:0.0:1788201706.601885:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477294592, bytes: 4096 00000080:00200000:0.0:1788201706.601887:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.601889:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.601890:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477294592, 477298688) 0 0 00000080:00200000:0.0:1788201706.601892:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.601895:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116527, 116527] 00020000:00200000:0.0:1788201706.601897:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477294592 chunk: [477294592, 477298688) 477298688, 4096 00020000:00200000:0.0:1788201706.601899:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.601905:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116527, 116527] 00000008:00010000:0.0:1788201706.601908:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.601910:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.601915:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601920:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477294592 477298687) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601926:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601930:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601934:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.601939:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477294592, 477298688) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.601942:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116527 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.601946:0:17463:0:(rw26.c:898:ll_write_end()) pos 477294592, len 4096, copied 4096 00000080:00200000:0.0:1788201706.601948:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001ca50354[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116527] 00000080:00200000:0.0:1788201706.601950:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003ef4bd31 fffffc0001029 2:0 ffff93c0c12d7c40 116527 lru 00000080:00200000:0.0:1788201706.601954:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e7705d96 116527: 1< 2 - - > 2< 477294592 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68112 - -> 00000080:00200000:0.0:1788201706.601955:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001ca50354 00000080:00200000:0.0:1788201706.601956:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.601957:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.601963:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.601970:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.601972:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.601973:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.601979:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.601981:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.601982:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.601985:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.601989:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.601994:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.601996:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.601998:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.602000:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116527, 116527] 00000080:00200000:0.0:1788201706.602007:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.602009:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.602044:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.602047:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.602058:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477298688, count: 4096 00000080:00200000:0.0:1788201706.602064:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477298688, bytes: 4096 00000080:00200000:0.0:1788201706.602065:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.602068:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.602069:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477298688, 477302784) 0 0 00000080:00200000:0.0:1788201706.602070:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.602074:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116528, 116528] 00020000:00200000:0.0:1788201706.602076:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477298688 chunk: [477298688, 477302784) 477302784, 4096 00020000:00200000:0.0:1788201706.602078:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.602084:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116528, 116528] 00000008:00010000:0.0:1788201706.602088:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.602089:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.602094:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602100:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477298688 477302783) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602106:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602110:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602114:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.602119:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477298688, 477302784) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.602122:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116528 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.602126:0:17463:0:(rw26.c:898:ll_write_end()) pos 477298688, len 4096, copied 4096 00000080:00200000:0.0:1788201706.602128:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ca5d7710[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116528] 00000080:00200000:0.0:1788201706.602130:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000011735073 fffffc0001029 2:0 ffff93c0c12d7540 116528 lru 00000080:00200000:0.0:1788201706.602134:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a7e3bbad 116528: 1< 2 - - > 2< 477298688 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68113 - -> 00000080:00200000:0.0:1788201706.602136:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ca5d7710 00000080:00200000:0.0:1788201706.602136:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.602137:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.602143:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.602151:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.602152:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.602154:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.602160:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.602161:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.602163:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.602165:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602169:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.602174:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.602176:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.602178:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.602180:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116528, 116528] 00000080:00200000:0.0:1788201706.602187:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.602189:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.602206:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.602208:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.602218:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477302784, count: 4096 00000080:00200000:0.0:1788201706.602224:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477302784, bytes: 4096 00000080:00200000:0.0:1788201706.602226:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.602228:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.602229:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477302784, 477306880) 0 0 00000080:00200000:0.0:1788201706.602231:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.602234:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116529, 116529] 00020000:00200000:0.0:1788201706.602236:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477302784 chunk: [477302784, 477306880) 477306880, 4096 00020000:00200000:0.0:1788201706.602237:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.602243:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116529, 116529] 00000008:00010000:0.0:1788201706.602246:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.602248:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.602254:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602262:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477302784 477306879) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602271:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602278:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602285:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.602299:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477302784, 477306880) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.602304:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116529 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.602310:0:17463:0:(rw26.c:898:ll_write_end()) pos 477302784, len 4096, copied 4096 00000080:00200000:0.0:1788201706.602312:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004d4db7c3[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116529] 00000080:00200000:0.0:1788201706.602314:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000297f09a9 fffffc0001029 2:0 ffff93c0c12d7460 116529 lru 00000080:00200000:0.0:1788201706.602318:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000a19e75e 116529: 1< 2 - - > 2< 477302784 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68114 - -> 00000080:00200000:0.0:1788201706.602319:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004d4db7c3 00000080:00200000:0.0:1788201706.602320:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.602321:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.602327:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.602335:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.602337:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.602338:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.602345:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.602346:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.602348:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.602350:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602356:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.602361:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.602362:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.602365:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.602367:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116529, 116529] 00000080:00200000:0.0:1788201706.602374:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.602376:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.602394:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.602396:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.602407:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477306880, count: 4096 00000080:00200000:0.0:1788201706.602413:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477306880, bytes: 4096 00000080:00200000:0.0:1788201706.602414:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.602417:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.602418:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477306880, 477310976) 0 0 00000080:00200000:0.0:1788201706.602420:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.602423:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116530, 116530] 00020000:00200000:0.0:1788201706.602425:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477306880 chunk: [477306880, 477310976) 477310976, 4096 00020000:00200000:0.0:1788201706.602426:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.602433:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116530, 116530] 00000008:00010000:0.0:1788201706.602436:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.602437:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.602443:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602448:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477306880 477310975) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602453:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602458:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602466:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.602470:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477306880, 477310976) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.602473:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116530 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.602476:0:17463:0:(rw26.c:898:ll_write_end()) pos 477306880, len 4096, copied 4096 00000080:00200000:0.0:1788201706.602478:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000586a0c52[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116530] 00000080:00200000:0.0:1788201706.602480:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001fa52458 fffffc0001029 2:0 ffff93c0c12d8d20 116530 lru 00000080:00200000:0.0:1788201706.602484:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007a7c13ac 116530: 1< 2 - - > 2< 477306880 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68115 - -> 00000080:00200000:0.0:1788201706.602485:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000586a0c52 00000080:00200000:0.0:1788201706.602486:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.602487:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.602495:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.602506:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.602509:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.602511:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.602520:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.602523:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.602525:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.602529:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602537:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.602549:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.602551:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.602555:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.602558:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116530, 116530] 00000080:00200000:0.0:1788201706.602568:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.602571:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.602597:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.602600:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.602615:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477310976, count: 4096 00000080:00200000:0.0:1788201706.602624:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477310976, bytes: 4096 00000080:00200000:0.0:1788201706.602627:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.602630:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.602632:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477310976, 477315072) 0 0 00000080:00200000:0.0:1788201706.602634:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.602639:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116531, 116531] 00020000:00200000:0.0:1788201706.602642:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477310976 chunk: [477310976, 477315072) 477315072, 4096 00020000:00200000:0.0:1788201706.602644:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.602654:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116531, 116531] 00000008:00010000:0.0:1788201706.602657:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.602659:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.602664:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602670:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477310976 477315071) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602676:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602684:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602688:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.602693:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477310976, 477315072) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.602695:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116531 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.602699:0:17463:0:(rw26.c:898:ll_write_end()) pos 477310976, len 4096, copied 4096 00000080:00200000:0.0:1788201706.602701:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000093ff2517[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116531] 00000080:00200000:0.0:1788201706.602703:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000090942cb7 fffffc0001029 2:0 ffff93c0c12d89a0 116531 lru 00000080:00200000:0.0:1788201706.602707:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ee1de2e5 116531: 1< 2 - - > 2< 477310976 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68116 - -> 00000080:00200000:0.0:1788201706.602708:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000093ff2517 00000080:00200000:0.0:1788201706.602708:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.602710:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.602715:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.602724:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.602725:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.602726:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.602733:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.602734:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.602736:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.602738:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602746:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.602751:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.602752:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.602755:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.602757:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116531, 116531] 00000080:00200000:0.0:1788201706.602763:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.602765:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.602783:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.602785:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.602795:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477315072, count: 4096 00000080:00200000:0.0:1788201706.602801:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477315072, bytes: 4096 00000080:00200000:0.0:1788201706.602803:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.602805:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.602806:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477315072, 477319168) 0 0 00000080:00200000:0.0:1788201706.602808:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.602811:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116532, 116532] 00020000:00200000:0.0:1788201706.602813:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477315072 chunk: [477315072, 477319168) 477319168, 4096 00020000:00200000:0.0:1788201706.602814:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.602820:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116532, 116532] 00000008:00010000:0.0:1788201706.602823:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.602825:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.602842:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602849:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477315072 477319167) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602858:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602863:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602867:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.602871:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477315072, 477319168) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.602874:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116532 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.602878:0:17463:0:(rw26.c:898:ll_write_end()) pos 477315072, len 4096, copied 4096 00000080:00200000:0.0:1788201706.602880:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e53e27d1[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116532] 00000080:00200000:0.0:1788201706.602882:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000025a1d203 fffffc0001029 2:0 ffff93c0c12d8b60 116532 lru 00000080:00200000:0.0:1788201706.602885:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003c6f3a03 116532: 1< 2 - - > 2< 477315072 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68117 - -> 00000080:00200000:0.0:1788201706.602886:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e53e27d1 00000080:00200000:0.0:1788201706.602887:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.602888:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.602894:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.602901:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.602903:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.602904:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.602910:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.602912:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.602914:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.602916:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.602921:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.602925:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.602927:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.602929:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.602931:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116532, 116532] 00000080:00200000:0.0:1788201706.602938:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.602940:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.602957:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.602959:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.602969:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477319168, count: 4096 00000080:00200000:0.0:1788201706.602975:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477319168, bytes: 4096 00000080:00200000:0.0:1788201706.602977:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.602979:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.602980:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477319168, 477323264) 0 0 00000080:00200000:0.0:1788201706.602981:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.602985:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116533, 116533] 00020000:00200000:0.0:1788201706.602986:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477319168 chunk: [477319168, 477323264) 477323264, 4096 00020000:00200000:0.0:1788201706.602988:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.602994:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116533, 116533] 00000008:00010000:0.0:1788201706.602997:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.602998:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.603003:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603012:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477319168 477323263) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603031:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603036:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603040:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.603044:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477319168, 477323264) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.603048:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116533 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.603051:0:17463:0:(rw26.c:898:ll_write_end()) pos 477319168, len 4096, copied 4096 00000080:00200000:0.0:1788201706.603054:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008535b0c9[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116533] 00000080:00200000:0.0:1788201706.603055:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000dff89f7c fffffc0001029 2:0 ffff93c0c12d8c40 116533 lru 00000080:00200000:0.0:1788201706.603059:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b9c76e87 116533: 1< 2 - - > 2< 477319168 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68118 - -> 00000080:00200000:0.0:1788201706.603060:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008535b0c9 00000080:00200000:0.0:1788201706.603061:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.603062:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.603068:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.603075:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.603077:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.603084:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.603091:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.603093:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.603094:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.603097:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603102:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.603107:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.603109:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.603111:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.603113:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116533, 116533] 00000080:00200000:0.0:1788201706.603120:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.603122:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.603140:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.603142:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.603152:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477323264, count: 4096 00000080:00200000:0.0:1788201706.603158:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477323264, bytes: 4096 00000080:00200000:0.0:1788201706.603160:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.603162:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.603163:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477323264, 477327360) 0 0 00000080:00200000:0.0:1788201706.603165:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.603168:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116534, 116534] 00020000:00200000:0.0:1788201706.603170:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477323264 chunk: [477323264, 477327360) 477327360, 4096 00020000:00200000:0.0:1788201706.603171:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.603178:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116534, 116534] 00000008:00010000:0.0:1788201706.603181:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.603182:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.603187:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603193:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477323264 477327359) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603198:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603203:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603207:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.603211:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477323264, 477327360) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.603215:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116534 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.603218:0:17463:0:(rw26.c:898:ll_write_end()) pos 477323264, len 4096, copied 4096 00000080:00200000:0.0:1788201706.603220:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001c098fad[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116534] 00000080:00200000:0.0:1788201706.603222:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d0b6f9db fffffc0001029 2:0 ffff93c0c12d8540 116534 lru 00000080:00200000:0.0:1788201706.603226:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000406247be 116534: 1< 2 - - > 2< 477323264 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68119 - -> 00000080:00200000:0.0:1788201706.603227:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001c098fad 00000080:00200000:0.0:1788201706.603228:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.603229:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.603235:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.603242:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.603244:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.603246:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.603253:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.603255:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.603258:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.603262:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603269:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.603277:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.603280:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.603284:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.603287:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116534, 116534] 00000080:00200000:0.0:1788201706.603297:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.603301:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.603323:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.603325:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.603336:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477327360, count: 4096 00000080:00200000:0.0:1788201706.603342:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477327360, bytes: 4096 00000080:00200000:0.0:1788201706.603344:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.603346:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.603347:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477327360, 477331456) 0 0 00000080:00200000:0.0:1788201706.603349:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.603352:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116535, 116535] 00020000:00200000:0.0:1788201706.603354:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477327360 chunk: [477327360, 477331456) 477331456, 4096 00020000:00200000:0.0:1788201706.603355:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.603362:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116535, 116535] 00000008:00010000:0.0:1788201706.603365:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.603367:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.603372:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603378:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477327360 477331455) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603383:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603388:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603392:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.603396:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477327360, 477331456) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.603400:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116535 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.603403:0:17463:0:(rw26.c:898:ll_write_end()) pos 477327360, len 4096, copied 4096 00000080:00200000:0.0:1788201706.603405:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a1c541f0[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116535] 00000080:00200000:0.0:1788201706.603407:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006df75f3f fffffc0001029 2:0 ffff93c0c12d8460 116535 lru 00000080:00200000:0.0:1788201706.603411:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000dc05c6ed 116535: 1< 2 - - > 2< 477327360 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68120 - -> 00000080:00200000:0.0:1788201706.603412:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a1c541f0 00000080:00200000:0.0:1788201706.603413:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.603414:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.603420:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.603431:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.603432:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.603434:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.603440:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.603441:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.603443:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.603445:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603450:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.603455:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.603456:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.603459:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.603461:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116535, 116535] 00000080:00200000:0.0:1788201706.603468:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.603470:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.603488:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.603491:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.603506:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477331456, count: 4096 00000080:00200000:0.0:1788201706.603516:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477331456, bytes: 4096 00000080:00200000:0.0:1788201706.603519:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.603523:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.603525:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477331456, 477335552) 0 0 00000080:00200000:0.0:1788201706.603527:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.603532:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116536, 116536] 00020000:00200000:0.0:1788201706.603536:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477331456 chunk: [477331456, 477335552) 477335552, 4096 00020000:00200000:0.0:1788201706.603538:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.603550:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116536, 116536] 00000008:00010000:0.0:1788201706.603554:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.603555:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.603561:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603567:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477331456 477335551) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603572:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603577:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603581:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.603586:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477331456, 477335552) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.603589:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116536 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.603593:0:17463:0:(rw26.c:898:ll_write_end()) pos 477331456, len 4096, copied 4096 00000080:00200000:0.0:1788201706.603595:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a77bb5be[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116536] 00000080:00200000:0.0:1788201706.603597:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fc47acfd fffffc0001029 2:0 ffff93c0c12d80e0 116536 lru 00000080:00200000:0.0:1788201706.603601:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c82d1710 116536: 1< 2 - - > 2< 477331456 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68121 - -> 00000080:00200000:0.0:1788201706.603602:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a77bb5be 00000080:00200000:0.0:1788201706.603603:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.603604:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.603610:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.603618:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.603620:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.603621:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.603627:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.603629:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.603630:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.603633:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603638:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.603643:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.603644:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.603646:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.603648:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116536, 116536] 00000080:00200000:0.0:1788201706.603655:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.603657:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.603675:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.603677:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.603688:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477335552, count: 4096 00000080:00200000:0.0:1788201706.603693:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477335552, bytes: 4096 00000080:00200000:0.0:1788201706.603695:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.603697:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.603698:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477335552, 477339648) 0 0 00000080:00200000:0.0:1788201706.603700:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.603703:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116537, 116537] 00020000:00200000:0.0:1788201706.603705:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477335552 chunk: [477335552, 477339648) 477339648, 4096 00020000:00200000:0.0:1788201706.603707:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.603713:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116537, 116537] 00000008:00010000:0.0:1788201706.603716:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.603717:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.603722:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603728:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477335552 477339647) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603733:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603738:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603742:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.603746:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477335552, 477339648) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.603749:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116537 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.603752:0:17463:0:(rw26.c:898:ll_write_end()) pos 477335552, len 4096, copied 4096 00000080:00200000:0.0:1788201706.603754:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000043885b81[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116537] 00000080:00200000:0.0:1788201706.603756:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000034e1371b fffffc0001029 2:0 ffff93c0c12d8700 116537 lru 00000080:00200000:0.0:1788201706.603759:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000045c69ca6 116537: 1< 2 - - > 2< 477335552 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68122 - -> 00000080:00200000:0.0:1788201706.603762:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000043885b81 00000080:00200000:0.0:1788201706.603762:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.603764:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.603769:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.603776:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.603778:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.603779:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.603785:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.603787:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.603788:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.603791:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603795:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.603799:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.603801:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.603803:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.603805:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116537, 116537] 00000080:00200000:0.0:1788201706.603812:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.603814:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.603844:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.603847:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.603858:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477339648, count: 4096 00000080:00200000:0.0:1788201706.603864:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477339648, bytes: 4096 00000080:00200000:0.0:1788201706.603866:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.603868:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.603869:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477339648, 477343744) 0 0 00000080:00200000:0.0:1788201706.603871:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.603874:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116538, 116538] 00020000:00200000:0.0:1788201706.603876:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477339648 chunk: [477339648, 477343744) 477343744, 4096 00020000:00200000:0.0:1788201706.603877:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.603884:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116538, 116538] 00000008:00010000:0.0:1788201706.603887:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.603888:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.603893:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603899:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477339648 477343743) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603905:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603910:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603913:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.603918:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477339648, 477343744) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.603921:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116538 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.603925:0:17463:0:(rw26.c:898:ll_write_end()) pos 477339648, len 4096, copied 4096 00000080:00200000:0.0:1788201706.603927:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004e42b16a[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116538] 00000080:00200000:0.0:1788201706.603929:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a8a9c59e fffffc0001029 2:0 ffff93c0c12d8000 116538 lru 00000080:00200000:0.0:1788201706.603933:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008acad229 116538: 1< 2 - - > 2< 477339648 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68123 - -> 00000080:00200000:0.0:1788201706.603934:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004e42b16a 00000080:00200000:0.0:1788201706.603935:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.603936:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.603941:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.603948:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.603950:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.603951:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.603957:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.603959:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.603961:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.603963:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.603968:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.603973:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.603975:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.603977:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.603979:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116538, 116538] 00000080:00200000:0.0:1788201706.603986:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.603987:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.604005:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.604007:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.604032:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477343744, count: 4096 00000080:00200000:0.0:1788201706.604040:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477343744, bytes: 4096 00000080:00200000:0.0:1788201706.604042:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.604044:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.604045:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477343744, 477347840) 0 0 00000080:00200000:0.0:1788201706.604047:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.604051:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116539, 116539] 00020000:00200000:0.0:1788201706.604052:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477343744 chunk: [477343744, 477347840) 477347840, 4096 00020000:00200000:0.0:1788201706.604054:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.604061:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116539, 116539] 00000008:00010000:0.0:1788201706.604064:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.604065:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.604070:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604076:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477343744 477347839) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604081:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604086:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604090:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.604095:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477343744, 477347840) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.604098:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116539 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.604102:0:17463:0:(rw26.c:898:ll_write_end()) pos 477343744, len 4096, copied 4096 00000080:00200000:0.0:1788201706.604104:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000032093205[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116539] 00000080:00200000:0.0:1788201706.604106:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000020e9563c fffffc0001029 2:0 ffff93c0c12d8a80 116539 lru 00000080:00200000:0.0:1788201706.604110:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000057f98914 116539: 1< 2 - - > 2< 477343744 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68124 - -> 00000080:00200000:0.0:1788201706.604111:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000032093205 00000080:00200000:0.0:1788201706.604112:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.604113:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.604119:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.604126:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.604128:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.604130:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.604136:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.604138:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.604139:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.604142:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604146:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.604152:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.604153:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.604156:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.604158:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116539, 116539] 00000080:00200000:0.0:1788201706.604164:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.604166:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.604184:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.604186:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.604196:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477347840, count: 4096 00000080:00200000:0.0:1788201706.604202:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477347840, bytes: 4096 00000080:00200000:0.0:1788201706.604204:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.604206:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.604207:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477347840, 477351936) 0 0 00000080:00200000:0.0:1788201706.604209:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.604212:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116540, 116540] 00020000:00200000:0.0:1788201706.604214:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477347840 chunk: [477347840, 477351936) 477351936, 4096 00020000:00200000:0.0:1788201706.604215:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.604221:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116540, 116540] 00000008:00010000:0.0:1788201706.604224:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.604226:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.604231:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604236:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477347840 477351935) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604242:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604246:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604252:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.604259:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477347840, 477351936) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.604264:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116540 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.604269:0:17463:0:(rw26.c:898:ll_write_end()) pos 477347840, len 4096, copied 4096 00000080:00200000:0.0:1788201706.604273:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006cdf780a[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116540] 00000080:00200000:0.0:1788201706.604276:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f3739af3 fffffc0001029 2:0 ffff93c0c12d8380 116540 lru 00000080:00200000:0.0:1788201706.604282:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ade82d4f 116540: 1< 2 - - > 2< 477347840 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68125 - -> 00000080:00200000:0.0:1788201706.604284:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006cdf780a 00000080:00200000:0.0:1788201706.604285:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.604287:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.604295:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.604307:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.604309:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.604310:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.604317:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.604318:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.604320:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.604323:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604328:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.604333:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.604334:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.604337:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.604339:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116540, 116540] 00000080:00200000:0.0:1788201706.604346:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.604348:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.604366:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.604368:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.604379:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477351936, count: 4096 00000080:00200000:0.0:1788201706.604385:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477351936, bytes: 4096 00000080:00200000:0.0:1788201706.604386:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.604389:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.604390:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477351936, 477356032) 0 0 00000080:00200000:0.0:1788201706.604391:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.604395:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116541, 116541] 00020000:00200000:0.0:1788201706.604397:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477351936 chunk: [477351936, 477356032) 477356032, 4096 00020000:00200000:0.0:1788201706.604398:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.604404:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116541, 116541] 00000008:00010000:0.0:1788201706.604408:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.604409:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.604414:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604420:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477351936 477356031) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604425:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604429:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604433:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.604441:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477351936, 477356032) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.604444:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116541 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.604448:0:17463:0:(rw26.c:898:ll_write_end()) pos 477351936, len 4096, copied 4096 00000080:00200000:0.0:1788201706.604450:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009e512734[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116541] 00000080:00200000:0.0:1788201706.604452:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a352c721 fffffc0001029 2:0 ffff93c0c12d88c0 116541 lru 00000080:00200000:0.0:1788201706.604456:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e6f72d4d 116541: 1< 2 - - > 2< 477351936 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68126 - -> 00000080:00200000:0.0:1788201706.604457:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009e512734 00000080:00200000:0.0:1788201706.604458:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.604459:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.604464:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.604472:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.604473:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.604474:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.604480:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.604482:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.604484:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.604486:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604493:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.604502:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.604504:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.604509:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.604512:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116541, 116541] 00000080:00200000:0.0:1788201706.604522:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.604525:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.604548:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.604550:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.604561:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477356032, count: 4096 00000080:00200000:0.0:1788201706.604567:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477356032, bytes: 4096 00000080:00200000:0.0:1788201706.604568:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.604571:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.604572:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477356032, 477360128) 0 0 00000080:00200000:0.0:1788201706.604573:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.604577:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116542, 116542] 00020000:00200000:0.0:1788201706.604579:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477356032 chunk: [477356032, 477360128) 477360128, 4096 00020000:00200000:0.0:1788201706.604580:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.604587:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116542, 116542] 00000008:00010000:0.0:1788201706.604590:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.604591:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.604597:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604602:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477356032 477360127) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604608:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604613:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604621:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.604625:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477356032, 477360128) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.604628:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116542 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.604631:0:17463:0:(rw26.c:898:ll_write_end()) pos 477356032, len 4096, copied 4096 00000080:00200000:0.0:1788201706.604634:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000330d0b18[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116542] 00000080:00200000:0.0:1788201706.604635:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b4851792 fffffc0001029 2:0 ffff93c0c12d8ee0 116542 lru 00000080:00200000:0.0:1788201706.604639:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000057ccdb16 116542: 1< 2 - - > 2< 477356032 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68127 - -> 00000080:00200000:0.0:1788201706.604640:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000330d0b18 00000080:00200000:0.0:1788201706.604641:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.604643:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.604648:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.604655:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.604657:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.604658:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.604664:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.604666:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.604667:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.604670:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604675:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.604684:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.604685:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.604688:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.604690:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116542, 116542] 00000080:00200000:0.0:1788201706.604696:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.604698:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.604716:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.604718:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.604728:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477360128, count: 4096 00000080:00200000:0.0:1788201706.604733:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477360128, bytes: 4096 00000080:00200000:0.0:1788201706.604744:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.604747:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.604748:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477360128, 477364224) 0 0 00000080:00200000:0.0:1788201706.604750:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.604753:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116543, 116543] 00020000:00200000:0.0:1788201706.604755:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477360128 chunk: [477360128, 477364224) 477364224, 4096 00020000:00200000:0.0:1788201706.604756:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.604763:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116543, 116543] 00000008:00010000:0.0:1788201706.604766:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.604768:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.604773:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604779:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477360128 477364223) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604788:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604797:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604801:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.604805:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477360128, 477364224) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.604808:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116543 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.604812:0:17463:0:(rw26.c:898:ll_write_end()) pos 477360128, len 4096, copied 4096 00000080:00200000:0.0:1788201706.604814:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c2ade1d5[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116543] 00000080:00200000:0.0:1788201706.604816:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c9a47f0c fffffc0001029 2:0 ffff93c0c12d87e0 116543 lru 00000080:00200000:0.0:1788201706.604819:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000fc016a10 116543: 1< 2 - - > 2< 477360128 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68128 - -> 00000080:00200000:0.0:1788201706.604820:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c2ade1d5 00000080:00200000:0.0:1788201706.604821:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.604822:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.604827:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.604849:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.604851:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.604852:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.604858:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.604860:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.604862:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.604864:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.604873:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.604878:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.604879:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.604882:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.604884:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116543, 116543] 00000080:00200000:0.0:1788201706.604891:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.604893:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.604921:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.604925:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.604941:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477364224, count: 4096 00000080:00200000:0.0:1788201706.604955:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477364224, bytes: 4096 00000080:00200000:0.0:1788201706.604960:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.604964:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.604965:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477364224, 477368320) 0 0 00000080:00200000:0.0:1788201706.604968:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.604973:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116544, 116544] 00020000:00200000:0.0:1788201706.604976:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477364224 chunk: [477364224, 477368320) 477368320, 4096 00020000:00200000:0.0:1788201706.604979:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.604990:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116544, 116544] 00000008:00010000:0.0:1788201706.604995:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.604997:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.605004:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605013:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477364224 477368319) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605039:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605046:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605053:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.605061:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477364224, 477368320) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.605067:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116544 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.605072:0:17463:0:(rw26.c:898:ll_write_end()) pos 477364224, len 4096, copied 4096 00000080:00200000:0.0:1788201706.605076:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004db93d9a[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116544] 00000080:00200000:0.0:1788201706.605079:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fea0ad67 fffffc0001029 2:0 ffff93c0c12d82a0 116544 lru 00000080:00200000:0.0:1788201706.605084:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003b897925 116544: 1< 2 - - > 2< 477364224 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68129 - -> 00000080:00200000:0.0:1788201706.605086:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004db93d9a 00000080:00200000:0.0:1788201706.605087:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.605089:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.605098:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.605109:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.605112:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.605114:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.605124:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.605127:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.605130:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.605133:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605141:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.605148:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.605150:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.605154:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.605157:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116544, 116544] 00000080:00200000:0.0:1788201706.605167:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.605170:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.605198:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.605202:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.605217:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477368320, count: 4096 00000080:00200000:0.0:1788201706.605228:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477368320, bytes: 4096 00000080:00200000:0.0:1788201706.605230:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.605234:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.605235:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477368320, 477372416) 0 0 00000080:00200000:0.0:1788201706.605238:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.605242:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116545, 116545] 00020000:00200000:0.0:1788201706.605245:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477368320 chunk: [477368320, 477372416) 477372416, 4096 00020000:00200000:0.0:1788201706.605248:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.605260:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116545, 116545] 00000008:00010000:0.0:1788201706.605265:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.605267:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.605275:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605288:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477368320 477372415) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605297:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605305:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605312:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.605320:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477368320, 477372416) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.605326:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116545 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.605331:0:17463:0:(rw26.c:898:ll_write_end()) pos 477368320, len 4096, copied 4096 00000080:00200000:0.0:1788201706.605335:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004e3fc648[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116545] 00000080:00200000:0.0:1788201706.605336:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004c0014e6 fffffc0001029 2:0 ffff93c0c12d8e00 116545 lru 00000080:00200000:0.0:1788201706.605340:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ecad88fd 116545: 1< 2 - - > 2< 477368320 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68130 - -> 00000080:00200000:0.0:1788201706.605342:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004e3fc648 00000080:00200000:0.0:1788201706.605342:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.605344:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.605350:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.605358:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.605360:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.605361:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.605368:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.605370:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.605372:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.605374:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605379:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.605384:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.605385:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.605388:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.605390:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116545, 116545] 00000080:00200000:0.0:1788201706.605399:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.605402:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.605431:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.605434:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.605450:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477372416, count: 4096 00000080:00200000:0.0:1788201706.605461:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477372416, bytes: 4096 00000080:00200000:0.0:1788201706.605464:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.605468:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.605469:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477372416, 477376512) 0 0 00000080:00200000:0.0:1788201706.605471:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.605476:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116546, 116546] 00020000:00200000:0.0:1788201706.605479:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477372416 chunk: [477372416, 477376512) 477376512, 4096 00020000:00200000:0.0:1788201706.605481:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.605493:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116546, 116546] 00000008:00010000:0.0:1788201706.605498:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.605500:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.605509:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605518:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477372416 477376511) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605526:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605534:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605540:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.605546:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477372416, 477376512) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.605550:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116546 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.605554:0:17463:0:(rw26.c:898:ll_write_end()) pos 477372416, len 4096, copied 4096 00000080:00200000:0.0:1788201706.605557:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000808f43af[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116546] 00000080:00200000:0.0:1788201706.605559:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f3188d5b fffffc0001029 2:0 ffff93c0c12d81c0 116546 lru 00000080:00200000:0.0:1788201706.605562:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000721b559b 116546: 1< 2 - - > 2< 477372416 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68131 - -> 00000080:00200000:0.0:1788201706.605564:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000808f43af 00000080:00200000:0.0:1788201706.605564:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.605566:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.605572:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.605580:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.605582:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.605583:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.605590:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.605592:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.605593:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.605596:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605601:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.605607:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.605608:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.605610:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.605612:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116546, 116546] 00000080:00200000:0.0:1788201706.605620:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.605622:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.605640:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.605642:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.605653:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477376512, count: 4096 00000080:00200000:0.0:1788201706.605659:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477376512, bytes: 4096 00000080:00200000:0.0:1788201706.605661:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.605663:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.605665:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477376512, 477380608) 0 0 00000080:00200000:0.0:1788201706.605666:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.605669:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116547, 116547] 00020000:00200000:0.0:1788201706.605671:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477376512 chunk: [477376512, 477380608) 477380608, 4096 00020000:00200000:0.0:1788201706.605673:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.605679:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116547, 116547] 00000008:00010000:0.0:1788201706.605683:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.605684:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.605689:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605695:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477376512 477380607) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605700:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605705:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605709:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.605713:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477376512, 477380608) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.605716:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116547 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.605720:0:17463:0:(rw26.c:898:ll_write_end()) pos 477376512, len 4096, copied 4096 00000080:00200000:0.0:1788201706.605722:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a4c1120b[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116547] 00000080:00200000:0.0:1788201706.605724:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003200b0ea fffffc0001029 2:0 ffff93c0c12d8620 116547 lru 00000080:00200000:0.0:1788201706.605728:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f7510300 116547: 1< 2 - - > 2< 477376512 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68132 - -> 00000080:00200000:0.0:1788201706.605729:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a4c1120b 00000080:00200000:0.0:1788201706.605730:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.605731:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.605736:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.605752:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.605754:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.605756:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.605765:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.605768:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.605770:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.605774:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605782:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.605790:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.605792:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.605797:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.605800:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116547, 116547] 00000080:00200000:0.0:1788201706.605811:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.605814:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.605860:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.605863:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.605879:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477380608, count: 4096 00000080:00200000:0.0:1788201706.605886:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477380608, bytes: 4096 00000080:00200000:0.0:1788201706.605888:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.605891:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.605892:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477380608, 477384704) 0 0 00000080:00200000:0.0:1788201706.605893:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.605897:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116548, 116548] 00020000:00200000:0.0:1788201706.605899:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477380608 chunk: [477380608, 477384704) 477384704, 4096 00020000:00200000:0.0:1788201706.605901:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.605908:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116548, 116548] 00000008:00010000:0.0:1788201706.605911:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.605913:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.605918:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605924:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477380608 477384703) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605929:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605934:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605938:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.605943:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477380608, 477384704) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.605946:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116548 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.605949:0:17463:0:(rw26.c:898:ll_write_end()) pos 477380608, len 4096, copied 4096 00000080:00200000:0.0:1788201706.605951:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000893c7f49[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116548] 00000080:00200000:0.0:1788201706.605953:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003983a8ac fffffc0001029 2:0 ffff93c0c12d9a80 116548 lru 00000080:00200000:0.0:1788201706.605957:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bff3e2c6 116548: 1< 2 - - > 2< 477380608 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68133 - -> 00000080:00200000:0.0:1788201706.605958:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000893c7f49 00000080:00200000:0.0:1788201706.605959:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.605961:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.605967:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.605975:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.605977:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.605978:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.605985:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.605986:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.605988:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.605990:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.605995:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.606000:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.606001:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.606004:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.606006:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116548, 116548] 00000080:00200000:0.0:1788201706.606012:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.606014:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.606058:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.606060:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.606071:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477384704, count: 4096 00000080:00200000:0.0:1788201706.606077:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477384704, bytes: 4096 00000080:00200000:0.0:1788201706.606079:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.606081:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.606082:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477384704, 477388800) 0 0 00000080:00200000:0.0:1788201706.606084:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.606087:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116549, 116549] 00020000:00200000:0.0:1788201706.606089:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477384704 chunk: [477384704, 477388800) 477388800, 4096 00020000:00200000:0.0:1788201706.606091:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.606097:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116549, 116549] 00000008:00010000:0.0:1788201706.606101:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.606102:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.606108:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606114:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477384704 477388799) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606120:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606124:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606128:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.606133:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477384704, 477388800) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.606136:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116549 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.606140:0:17463:0:(rw26.c:898:ll_write_end()) pos 477384704, len 4096, copied 4096 00000080:00200000:0.0:1788201706.606142:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003e7c87f5[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116549] 00000080:00200000:0.0:1788201706.606144:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005c639786 fffffc0001029 2:0 ffff93c0c12d9380 116549 lru 00000080:00200000:0.0:1788201706.606148:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003506bcaa 116549: 1< 2 - - > 2< 477384704 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68134 - -> 00000080:00200000:0.0:1788201706.606151:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003e7c87f5 00000080:00200000:0.0:1788201706.606151:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.606153:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.606158:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.606166:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.606168:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.606169:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.606175:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.606177:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.606179:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.606181:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606186:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.606191:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.606192:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.606194:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.606196:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116549, 116549] 00000080:00200000:0.0:1788201706.606203:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.606205:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.606222:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.606225:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.606236:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477388800, count: 4096 00000080:00200000:0.0:1788201706.606246:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477388800, bytes: 4096 00000080:00200000:0.0:1788201706.606250:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.606253:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.606255:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477388800, 477392896) 0 0 00000080:00200000:0.0:1788201706.606257:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.606261:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116550, 116550] 00020000:00200000:0.0:1788201706.606264:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477388800 chunk: [477388800, 477392896) 477392896, 4096 00020000:00200000:0.0:1788201706.606266:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.606278:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116550, 116550] 00000008:00010000:0.0:1788201706.606284:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.606286:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.606294:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606303:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477388800 477392895) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606311:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606318:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606324:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.606331:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477388800, 477392896) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.606335:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116550 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.606340:0:17463:0:(rw26.c:898:ll_write_end()) pos 477388800, len 4096, copied 4096 00000080:00200000:0.0:1788201706.606344:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a36304e6[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116550] 00000080:00200000:0.0:1788201706.606347:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005e23978b fffffc0001029 2:0 ffff93c0c12d98c0 116550 lru 00000080:00200000:0.0:1788201706.606352:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009cb8604a 116550: 1< 2 - - > 2< 477388800 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68135 - -> 00000080:00200000:0.0:1788201706.606354:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a36304e6 00000080:00200000:0.0:1788201706.606355:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.606357:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.606366:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.606378:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.606380:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.606383:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.606390:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.606392:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.606394:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.606396:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606401:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.606406:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.606408:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.606410:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.606412:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116550, 116550] 00000080:00200000:0.0:1788201706.606420:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.606421:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.606440:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.606442:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.606454:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477392896, count: 4096 00000080:00200000:0.0:1788201706.606460:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477392896, bytes: 4096 00000080:00200000:0.0:1788201706.606461:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.606463:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.606464:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477392896, 477396992) 0 0 00000080:00200000:0.0:1788201706.606466:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.606469:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116551, 116551] 00020000:00200000:0.0:1788201706.606471:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477392896 chunk: [477392896, 477396992) 477396992, 4096 00020000:00200000:0.0:1788201706.606473:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.606479:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116551, 116551] 00000008:00010000:0.0:1788201706.606482:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.606484:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.606490:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606499:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477392896 477396991) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606508:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606516:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606523:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.606530:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477392896, 477396992) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.606535:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116551 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.606541:0:17463:0:(rw26.c:898:ll_write_end()) pos 477392896, len 4096, copied 4096 00000080:00200000:0.0:1788201706.606544:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e1ba587f[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116551] 00000080:00200000:0.0:1788201706.606547:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000007ebf62d fffffc0001029 2:0 ffff93c0c12d9ee0 116551 lru 00000080:00200000:0.0:1788201706.606552:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e21d580d 116551: 1< 2 - - > 2< 477392896 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68136 - -> 00000080:00200000:0.0:1788201706.606554:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e1ba587f 00000080:00200000:0.0:1788201706.606555:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.606557:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.606565:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.606576:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.606579:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.606581:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.606590:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.606593:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.606595:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.606599:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606607:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.606614:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.606615:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.606618:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.606620:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116551, 116551] 00000080:00200000:0.0:1788201706.606628:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.606630:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.606649:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.606651:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.606662:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477396992, count: 4096 00000080:00200000:0.0:1788201706.606668:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477396992, bytes: 4096 00000080:00200000:0.0:1788201706.606670:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.606672:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.606673:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477396992, 477401088) 0 0 00000080:00200000:0.0:1788201706.606674:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.606678:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116552, 116552] 00020000:00200000:0.0:1788201706.606680:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477396992 chunk: [477396992, 477401088) 477401088, 4096 00020000:00200000:0.0:1788201706.606681:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.606688:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116552, 116552] 00000008:00010000:0.0:1788201706.606691:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.606692:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.606698:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606703:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477396992 477401087) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606709:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606713:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606717:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.606721:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477396992, 477401088) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.606725:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116552 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.606728:0:17463:0:(rw26.c:898:ll_write_end()) pos 477396992, len 4096, copied 4096 00000080:00200000:0.0:1788201706.606731:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000081370416[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116552] 00000080:00200000:0.0:1788201706.606732:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f6b6f651 fffffc0001029 2:0 ffff93c0c12d97e0 116552 lru 00000080:00200000:0.0:1788201706.606736:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000eb39229f 116552: 1< 2 - - > 2< 477396992 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68137 - -> 00000080:00200000:0.0:1788201706.606738:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000081370416 00000080:00200000:0.0:1788201706.606738:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.606739:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.606745:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.606753:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.606754:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.606756:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.606762:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.606764:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.606765:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.606768:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606773:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.606777:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.606778:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.606781:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.606783:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116552, 116552] 00000080:00200000:0.0:1788201706.606789:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.606791:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.606809:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.606811:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.606821:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477401088, count: 4096 00000080:00200000:0.0:1788201706.606827:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477401088, bytes: 4096 00000080:00200000:0.0:1788201706.606841:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.606844:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.606845:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477401088, 477405184) 0 0 00000080:00200000:0.0:1788201706.606847:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.606850:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116553, 116553] 00020000:00200000:0.0:1788201706.606852:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477401088 chunk: [477401088, 477405184) 477405184, 4096 00020000:00200000:0.0:1788201706.606853:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.606861:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116553, 116553] 00000008:00010000:0.0:1788201706.606864:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.606865:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.606870:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606876:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477401088 477405183) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606881:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606886:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606890:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.606898:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477401088, 477405184) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.606901:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116553 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.606905:0:17463:0:(rw26.c:898:ll_write_end()) pos 477401088, len 4096, copied 4096 00000080:00200000:0.0:1788201706.606907:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000083b7694b[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116553] 00000080:00200000:0.0:1788201706.606909:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009fb2aa41 fffffc0001029 2:0 ffff93c0c12d92a0 116553 lru 00000080:00200000:0.0:1788201706.606913:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000db5ed62a 116553: 1< 2 - - > 2< 477401088 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68138 - -> 00000080:00200000:0.0:1788201706.606914:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000083b7694b 00000080:00200000:0.0:1788201706.606915:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.606916:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.606921:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.606928:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.606930:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.606931:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.606937:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.606939:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.606941:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.606943:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.606948:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.606952:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.606954:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.606957:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.606958:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116553, 116553] 00000080:00200000:0.0:1788201706.606965:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.606967:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.606984:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.606986:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.606996:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477405184, count: 4096 00000080:00200000:0.0:1788201706.607002:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477405184, bytes: 4096 00000080:00200000:0.0:1788201706.607004:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.607006:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.607007:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477405184, 477409280) 0 0 00000080:00200000:0.0:1788201706.607009:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.607012:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116554, 116554] 00020000:00200000:0.0:1788201706.607014:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477405184 chunk: [477405184, 477409280) 477409280, 4096 00020000:00200000:0.0:1788201706.607029:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.607041:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116554, 116554] 00000008:00010000:0.0:1788201706.607046:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.607048:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.607055:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607061:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477405184 477409279) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607066:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607071:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607078:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.607083:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477405184, 477409280) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.607086:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116554 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.607090:0:17463:0:(rw26.c:898:ll_write_end()) pos 477405184, len 4096, copied 4096 00000080:00200000:0.0:1788201706.607092:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000098044474[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116554] 00000080:00200000:0.0:1788201706.607094:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000967e878f fffffc0001029 2:0 ffff93c0c12d9e00 116554 lru 00000080:00200000:0.0:1788201706.607097:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007b48d815 116554: 1< 2 - - > 2< 477405184 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68139 - -> 00000080:00200000:0.0:1788201706.607099:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000098044474 00000080:00200000:0.0:1788201706.607099:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.607101:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.607106:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.607114:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.607115:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.607117:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.607123:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.607125:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.607126:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.607129:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607133:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.607142:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.607143:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.607146:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.607148:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116554, 116554] 00000080:00200000:0.0:1788201706.607155:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.607157:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.607175:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.607177:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.607187:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477409280, count: 4096 00000080:00200000:0.0:1788201706.607193:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477409280, bytes: 4096 00000080:00200000:0.0:1788201706.607194:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.607197:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.607198:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477409280, 477413376) 0 0 00000080:00200000:0.0:1788201706.607199:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.607203:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116555, 116555] 00020000:00200000:0.0:1788201706.607204:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477409280 chunk: [477409280, 477413376) 477413376, 4096 00020000:00200000:0.0:1788201706.607206:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.607213:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116555, 116555] 00000008:00010000:0.0:1788201706.607215:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.607217:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.607221:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607227:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477409280 477413375) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607233:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607240:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607245:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.607249:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477409280, 477413376) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.607251:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116555 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.607255:0:17463:0:(rw26.c:898:ll_write_end()) pos 477409280, len 4096, copied 4096 00000080:00200000:0.0:1788201706.607257:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000039cc57a7[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116555] 00000080:00200000:0.0:1788201706.607259:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fece71a7 fffffc0001029 2:0 ffff93c0c12d91c0 116555 lru 00000080:00200000:0.0:1788201706.607262:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000fc295ef8 116555: 1< 2 - - > 2< 477409280 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68140 - -> 00000080:00200000:0.0:1788201706.607263:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000039cc57a7 00000080:00200000:0.0:1788201706.607264:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.607265:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.607270:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.607278:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.607280:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.607281:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.607287:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.607288:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.607290:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.607292:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607301:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.607306:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.607307:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.607309:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.607311:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116555, 116555] 00000080:00200000:0.0:1788201706.607317:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.607319:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.607337:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.607339:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.607349:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477413376, count: 4096 00000080:00200000:0.0:1788201706.607355:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477413376, bytes: 4096 00000080:00200000:0.0:1788201706.607356:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.607359:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.607360:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477413376, 477417472) 0 0 00000080:00200000:0.0:1788201706.607361:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.607364:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116556, 116556] 00020000:00200000:0.0:1788201706.607366:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477413376 chunk: [477413376, 477417472) 477417472, 4096 00020000:00200000:0.0:1788201706.607368:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.607374:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116556, 116556] 00000008:00010000:0.0:1788201706.607377:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.607378:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.607383:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607389:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477413376 477417471) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607397:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607402:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607405:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.607410:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477413376, 477417472) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.607413:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116556 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.607417:0:17463:0:(rw26.c:898:ll_write_end()) pos 477413376, len 4096, copied 4096 00000080:00200000:0.0:1788201706.607419:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000062bea5b9[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116556] 00000080:00200000:0.0:1788201706.607421:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000392f1cb0 fffffc0001029 2:0 ffff93c0c12d9620 116556 lru 00000080:00200000:0.0:1788201706.607425:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a5a6f071 116556: 1< 2 - - > 2< 477413376 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68141 - -> 00000080:00200000:0.0:1788201706.607426:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000062bea5b9 00000080:00200000:0.0:1788201706.607427:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.607428:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.607433:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.607441:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.607443:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.607444:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.607450:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.607451:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.607453:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.607456:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607460:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.607465:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.607467:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.607469:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.607471:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116556, 116556] 00000080:00200000:0.0:1788201706.607477:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.607479:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.607496:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.607499:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.607511:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477417472, count: 4096 00000080:00200000:0.0:1788201706.607521:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477417472, bytes: 4096 00000080:00200000:0.0:1788201706.607524:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.607527:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.607529:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477417472, 477421568) 0 0 00000080:00200000:0.0:1788201706.607531:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.607536:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116557, 116557] 00020000:00200000:0.0:1788201706.607539:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477417472 chunk: [477417472, 477421568) 477421568, 4096 00020000:00200000:0.0:1788201706.607541:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.607551:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116557, 116557] 00000008:00010000:0.0:1788201706.607555:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.607556:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.607561:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607570:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477417472 477421567) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607576:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607580:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607584:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.607589:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477417472, 477421568) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.607592:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116557 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.607595:0:17463:0:(rw26.c:898:ll_write_end()) pos 477417472, len 4096, copied 4096 00000080:00200000:0.0:1788201706.607598:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e5e002c3[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116557] 00000080:00200000:0.0:1788201706.607600:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c82b6e9c fffffc0001029 2:0 ffff93c0c12d9d20 116557 lru 00000080:00200000:0.0:1788201706.607603:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007ced6c38 116557: 1< 2 - - > 2< 477417472 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68142 - -> 00000080:00200000:0.0:1788201706.607605:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e5e002c3 00000080:00200000:0.0:1788201706.607606:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.607607:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.607612:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.607620:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.607622:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.607623:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.607629:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.607631:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.607633:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.607635:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607640:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.607644:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.607646:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.607648:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.607650:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116557, 116557] 00000080:00200000:0.0:1788201706.607657:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.607659:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.607677:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.607679:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.607690:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477421568, count: 4096 00000080:00200000:0.0:1788201706.607696:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477421568, bytes: 4096 00000080:00200000:0.0:1788201706.607697:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.607700:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.607701:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477421568, 477425664) 0 0 00000080:00200000:0.0:1788201706.607702:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.607705:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116558, 116558] 00020000:00200000:0.0:1788201706.607707:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477421568 chunk: [477421568, 477425664) 477425664, 4096 00020000:00200000:0.0:1788201706.607709:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.607715:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116558, 116558] 00000008:00010000:0.0:1788201706.607718:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.607719:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.607725:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607730:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477421568 477425663) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607736:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607740:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607744:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.607748:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477421568, 477425664) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.607751:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116558 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.607755:0:17463:0:(rw26.c:898:ll_write_end()) pos 477421568, len 4096, copied 4096 00000080:00200000:0.0:1788201706.607757:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b2b24036[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116558] 00000080:00200000:0.0:1788201706.607759:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000700e9eb1 fffffc0001029 2:0 ffff93c0c12d99a0 116558 lru 00000080:00200000:0.0:1788201706.607762:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f4f33a65 116558: 1< 2 - - > 2< 477421568 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68143 - -> 00000080:00200000:0.0:1788201706.607764:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b2b24036 00000080:00200000:0.0:1788201706.607765:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.607766:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.607771:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.607778:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.607780:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.607782:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.607787:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.607789:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.607791:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.607793:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607797:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.607802:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.607803:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.607806:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.607808:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116558, 116558] 00000080:00200000:0.0:1788201706.607814:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.607816:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.607851:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.607854:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.607864:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477425664, count: 4096 00000080:00200000:0.0:1788201706.607871:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477425664, bytes: 4096 00000080:00200000:0.0:1788201706.607873:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.607875:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.607876:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477425664, 477429760) 0 0 00000080:00200000:0.0:1788201706.607877:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.607881:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116559, 116559] 00020000:00200000:0.0:1788201706.607883:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477425664 chunk: [477425664, 477429760) 477429760, 4096 00020000:00200000:0.0:1788201706.607884:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.607891:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116559, 116559] 00000008:00010000:0.0:1788201706.607895:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.607896:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.607901:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607907:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477425664 477429759) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607913:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607917:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607921:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.607925:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477425664, 477429760) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.607928:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116559 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.607932:0:17463:0:(rw26.c:898:ll_write_end()) pos 477425664, len 4096, copied 4096 00000080:00200000:0.0:1788201706.607934:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000038977201[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116559] 00000080:00200000:0.0:1788201706.607936:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000051a685ce fffffc0001029 2:0 ffff93c0c12d9b60 116559 lru 00000080:00200000:0.0:1788201706.607940:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a0de003f 116559: 1< 2 - - > 2< 477425664 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68144 - -> 00000080:00200000:0.0:1788201706.607941:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000038977201 00000080:00200000:0.0:1788201706.607942:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.607943:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.607948:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.607959:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.607961:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.607962:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.607968:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.607970:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.607972:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.607974:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.607978:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.607983:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.607985:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.607988:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.607989:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116559, 116559] 00000080:00200000:0.0:1788201706.607996:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.607998:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.608027:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.608030:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.608041:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477429760, count: 4096 00000080:00200000:0.0:1788201706.608047:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477429760, bytes: 4096 00000080:00200000:0.0:1788201706.608049:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.608051:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.608052:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477429760, 477433856) 0 0 00000080:00200000:0.0:1788201706.608054:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.608057:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116560, 116560] 00020000:00200000:0.0:1788201706.608059:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477429760 chunk: [477429760, 477433856) 477433856, 4096 00020000:00200000:0.0:1788201706.608061:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.608067:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116560, 116560] 00000008:00010000:0.0:1788201706.608070:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.608071:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.608076:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608082:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477429760 477433855) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608088:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608092:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608096:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.608101:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477429760, 477433856) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.608104:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116560 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.608108:0:17463:0:(rw26.c:898:ll_write_end()) pos 477429760, len 4096, copied 4096 00000080:00200000:0.0:1788201706.608110:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007662f0d9[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116560] 00000080:00200000:0.0:1788201706.608112:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000228f9c79 fffffc0001029 2:0 ffff93c0c12d9c40 116560 lru 00000080:00200000:0.0:1788201706.608116:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000699dd5ec 116560: 1< 2 - - > 2< 477429760 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68145 - -> 00000080:00200000:0.0:1788201706.608117:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007662f0d9 00000080:00200000:0.0:1788201706.608118:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.608119:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.608125:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.608132:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.608134:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.608135:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.608141:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.608143:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.608145:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.608147:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608152:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.608157:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.608158:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.608160:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.608162:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116560, 116560] 00000080:00200000:0.0:1788201706.608168:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.608170:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.608188:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.608190:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.608200:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477433856, count: 4096 00000080:00200000:0.0:1788201706.608206:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477433856, bytes: 4096 00000080:00200000:0.0:1788201706.608208:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.608210:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.608211:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477433856, 477437952) 0 0 00000080:00200000:0.0:1788201706.608213:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.608216:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116561, 116561] 00020000:00200000:0.0:1788201706.608217:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477433856 chunk: [477433856, 477437952) 477437952, 4096 00020000:00200000:0.0:1788201706.608219:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.608225:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116561, 116561] 00000008:00010000:0.0:1788201706.608228:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.608229:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.608234:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608240:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477433856 477437951) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608245:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608250:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608253:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.608258:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477433856, 477437952) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.608261:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116561 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.608265:0:17463:0:(rw26.c:898:ll_write_end()) pos 477433856, len 4096, copied 4096 00000080:00200000:0.0:1788201706.608267:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000503cca88[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116561] 00000080:00200000:0.0:1788201706.608269:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007ceb0f47 fffffc0001029 2:0 ffff93c0c12d9540 116561 lru 00000080:00200000:0.0:1788201706.608273:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000072b36b64 116561: 1< 2 - - > 2< 477433856 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68146 - -> 00000080:00200000:0.0:1788201706.608276:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000503cca88 00000080:00200000:0.0:1788201706.608276:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.608277:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.608283:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.608290:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.608292:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.608293:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.608299:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.608301:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.608302:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.608305:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608309:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.608314:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.608315:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.608318:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.608320:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116561, 116561] 00000080:00200000:0.0:1788201706.608326:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.608328:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.608345:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.608347:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.608357:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477437952, count: 4096 00000080:00200000:0.0:1788201706.608363:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477437952, bytes: 4096 00000080:00200000:0.0:1788201706.608365:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.608367:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.608368:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477437952, 477442048) 0 0 00000080:00200000:0.0:1788201706.608370:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.608373:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116562, 116562] 00020000:00200000:0.0:1788201706.608375:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477437952 chunk: [477437952, 477442048) 477442048, 4096 00020000:00200000:0.0:1788201706.608376:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.608382:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116562, 116562] 00000008:00010000:0.0:1788201706.608385:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.608387:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.608391:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608397:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477437952 477442047) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608402:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608407:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608411:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.608415:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477437952, 477442048) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.608417:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116562 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.608421:0:17463:0:(rw26.c:898:ll_write_end()) pos 477437952, len 4096, copied 4096 00000080:00200000:0.0:1788201706.608423:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009083a2a9[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116562] 00000080:00200000:0.0:1788201706.608425:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000dbbb4b84 fffffc0001029 2:0 ffff93c0c12d9460 116562 lru 00000080:00200000:0.0:1788201706.608429:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006750c220 116562: 1< 2 - - > 2< 477437952 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68147 - -> 00000080:00200000:0.0:1788201706.608431:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009083a2a9 00000080:00200000:0.0:1788201706.608431:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.608432:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.608437:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.608444:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.608446:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.608448:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.608454:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.608455:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.608457:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.608459:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608464:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.608468:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.608470:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.608472:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.608474:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116562, 116562] 00000080:00200000:0.0:1788201706.608480:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.608482:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.608500:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.608503:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.608518:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477442048, count: 4096 00000080:00200000:0.0:1788201706.608528:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477442048, bytes: 4096 00000080:00200000:0.0:1788201706.608531:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.608535:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.608537:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477442048, 477446144) 0 0 00000080:00200000:0.0:1788201706.608539:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.608543:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116563, 116563] 00020000:00200000:0.0:1788201706.608546:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477442048 chunk: [477442048, 477446144) 477446144, 4096 00020000:00200000:0.0:1788201706.608548:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.608556:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116563, 116563] 00000008:00010000:0.0:1788201706.608559:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.608560:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.608566:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608571:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477442048 477446143) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608577:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608581:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608585:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.608590:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477442048, 477446144) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.608593:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116563 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.608597:0:17463:0:(rw26.c:898:ll_write_end()) pos 477442048, len 4096, copied 4096 00000080:00200000:0.0:1788201706.608600:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000507d5325[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116563] 00000080:00200000:0.0:1788201706.608602:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008daa1a59 fffffc0001029 2:0 ffff93c0c12d90e0 116563 lru 00000080:00200000:0.0:1788201706.608605:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006f23fe1b 116563: 1< 2 - - > 2< 477442048 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68148 - -> 00000080:00200000:0.0:1788201706.608607:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000507d5325 00000080:00200000:0.0:1788201706.608607:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.608609:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.608615:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.608622:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.608624:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.608625:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.608631:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.608633:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.608635:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.608637:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608642:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.608646:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.608648:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.608650:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.608652:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116563, 116563] 00000080:00200000:0.0:1788201706.608659:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.608661:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.608679:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.608681:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.608691:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477446144, count: 4096 00000080:00200000:0.0:1788201706.608697:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477446144, bytes: 4096 00000080:00200000:0.0:1788201706.608699:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.608701:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.608702:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477446144, 477450240) 0 0 00000080:00200000:0.0:1788201706.608704:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.608707:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116564, 116564] 00020000:00200000:0.0:1788201706.608709:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477446144 chunk: [477446144, 477450240) 477450240, 4096 00020000:00200000:0.0:1788201706.608710:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.608716:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116564, 116564] 00000008:00010000:0.0:1788201706.608719:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.608720:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.608725:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608731:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477446144 477450239) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608736:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608741:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608744:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.608749:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477446144, 477450240) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.608752:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116564 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.608756:0:17463:0:(rw26.c:898:ll_write_end()) pos 477446144, len 4096, copied 4096 00000080:00200000:0.0:1788201706.608758:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000cce40d20[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116564] 00000080:00200000:0.0:1788201706.608760:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000850472f9 fffffc0001029 2:0 ffff93c0c12d9700 116564 lru 00000080:00200000:0.0:1788201706.608764:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000068481173 116564: 1< 2 - - > 2< 477446144 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68149 - -> 00000080:00200000:0.0:1788201706.608765:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000cce40d20 00000080:00200000:0.0:1788201706.608766:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.608767:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.608773:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.608780:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.608782:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.608783:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.608789:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.608790:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.608792:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.608794:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608799:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.608804:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.608805:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.608808:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.608810:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116564, 116564] 00000080:00200000:0.0:1788201706.608816:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.608818:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.608850:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.608853:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.608863:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477450240, count: 4096 00000080:00200000:0.0:1788201706.608869:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477450240, bytes: 4096 00000080:00200000:0.0:1788201706.608871:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.608873:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.608874:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477450240, 477454336) 0 0 00000080:00200000:0.0:1788201706.608876:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.608879:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116565, 116565] 00020000:00200000:0.0:1788201706.608881:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477450240 chunk: [477450240, 477454336) 477454336, 4096 00020000:00200000:0.0:1788201706.608882:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.608888:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116565, 116565] 00000008:00010000:0.0:1788201706.608891:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.608893:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.608898:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608903:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477450240 477454335) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608909:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608913:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608917:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.608925:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477450240, 477454336) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.608928:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116565 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.608932:0:17463:0:(rw26.c:898:ll_write_end()) pos 477450240, len 4096, copied 4096 00000080:00200000:0.0:1788201706.608934:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e55fc227[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116565] 00000080:00200000:0.0:1788201706.608936:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000054018e99 fffffc0001029 2:0 ffff93c0c12d9000 116565 lru 00000080:00200000:0.0:1788201706.608939:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000004266405 116565: 1< 2 - - > 2< 477450240 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68150 - -> 00000080:00200000:0.0:1788201706.608941:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e55fc227 00000080:00200000:0.0:1788201706.608941:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.608943:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.608948:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.608955:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.608957:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.608958:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.608964:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.608966:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.608968:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.608970:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.608974:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.608979:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.608980:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.608983:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.608985:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116565, 116565] 00000080:00200000:0.0:1788201706.608991:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.608993:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.609011:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.609013:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.609037:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477454336, count: 4096 00000080:00200000:0.0:1788201706.609044:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477454336, bytes: 4096 00000080:00200000:0.0:1788201706.609046:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.609049:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.609050:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477454336, 477458432) 0 0 00000080:00200000:0.0:1788201706.609051:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.609054:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116566, 116566] 00020000:00200000:0.0:1788201706.609057:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477454336 chunk: [477454336, 477458432) 477458432, 4096 00020000:00200000:0.0:1788201706.609058:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.609064:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116566, 116566] 00000008:00010000:0.0:1788201706.609068:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.609069:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.609075:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609080:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477454336 477458431) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609085:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609090:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609097:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.609102:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477454336, 477458432) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.609105:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116566 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.609108:0:17463:0:(rw26.c:898:ll_write_end()) pos 477454336, len 4096, copied 4096 00000080:00200000:0.0:1788201706.609110:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000396f940d[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116566] 00000080:00200000:0.0:1788201706.609112:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005f974f06 fffffc0001029 2:0 ffff93c0c12da620 116566 lru 00000080:00200000:0.0:1788201706.609116:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e72a41c3 116566: 1< 2 - - > 2< 477454336 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68151 - -> 00000080:00200000:0.0:1788201706.609117:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000396f940d 00000080:00200000:0.0:1788201706.609118:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.609119:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.609124:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.609132:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.609134:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.609135:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.609141:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.609143:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.609144:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.609147:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609152:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.609160:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.609162:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.609164:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.609166:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116566, 116566] 00000080:00200000:0.0:1788201706.609173:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.609175:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.609193:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.609195:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.609205:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477458432, count: 4096 00000080:00200000:0.0:1788201706.609211:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477458432, bytes: 4096 00000080:00200000:0.0:1788201706.609212:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.609214:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.609215:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477458432, 477462528) 0 0 00000080:00200000:0.0:1788201706.609217:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.609220:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116567, 116567] 00020000:00200000:0.0:1788201706.609222:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477458432 chunk: [477458432, 477462528) 477462528, 4096 00020000:00200000:0.0:1788201706.609223:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.609229:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116567, 116567] 00000008:00010000:0.0:1788201706.609232:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.609234:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.609238:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609244:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477458432 477462527) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609250:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609258:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609262:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.609266:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477458432, 477462528) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.609268:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116567 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.609272:0:17463:0:(rw26.c:898:ll_write_end()) pos 477458432, len 4096, copied 4096 00000080:00200000:0.0:1788201706.609274:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000049bce35d[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116567] 00000080:00200000:0.0:1788201706.609276:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ef8424bb fffffc0001029 2:0 ffff93c0c12dad20 116567 lru 00000080:00200000:0.0:1788201706.609279:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b46c0d7d 116567: 1< 2 - - > 2< 477458432 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68152 - -> 00000080:00200000:0.0:1788201706.609280:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000049bce35d 00000080:00200000:0.0:1788201706.609281:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.609282:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.609288:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.609295:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.609297:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.609298:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.609304:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.609306:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.609307:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.609310:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609318:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.609323:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.609324:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.609326:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.609328:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116567, 116567] 00000080:00200000:0.0:1788201706.609335:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.609336:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.609354:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.609356:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.609366:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477462528, count: 4096 00000080:00200000:0.0:1788201706.609372:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477462528, bytes: 4096 00000080:00200000:0.0:1788201706.609373:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.609375:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.609376:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477462528, 477466624) 0 0 00000080:00200000:0.0:1788201706.609378:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.609381:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116568, 116568] 00020000:00200000:0.0:1788201706.609383:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477462528 chunk: [477462528, 477466624) 477466624, 4096 00020000:00200000:0.0:1788201706.609384:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.609390:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116568, 116568] 00000008:00010000:0.0:1788201706.609393:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.609395:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.609399:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609405:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477462528 477466623) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609414:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609419:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609423:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.609427:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477462528, 477466624) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.609429:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116568 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.609433:0:17463:0:(rw26.c:898:ll_write_end()) pos 477462528, len 4096, copied 4096 00000080:00200000:0.0:1788201706.609435:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000174452fa[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116568] 00000080:00200000:0.0:1788201706.609437:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008c2ea548 fffffc0001029 2:0 ffff93c0c12da9a0 116568 lru 00000080:00200000:0.0:1788201706.609440:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001ada9bc7 116568: 1< 2 - - > 2< 477462528 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68153 - -> 00000080:00200000:0.0:1788201706.609441:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000174452fa 00000080:00200000:0.0:1788201706.609442:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.609443:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.609449:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.609456:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.609458:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.609459:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.609465:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.609467:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.609469:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.609471:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609476:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.609480:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.609482:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.609484:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.609486:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116568, 116568] 00000080:00200000:0.0:1788201706.609493:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.609495:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.609519:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.609523:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.609539:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477466624, count: 4096 00000080:00200000:0.0:1788201706.609550:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477466624, bytes: 4096 00000080:00200000:0.0:1788201706.609552:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.609554:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.609555:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477466624, 477470720) 0 0 00000080:00200000:0.0:1788201706.609557:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.609560:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116569, 116569] 00020000:00200000:0.0:1788201706.609562:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477466624 chunk: [477466624, 477470720) 477470720, 4096 00020000:00200000:0.0:1788201706.609564:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.609570:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116569, 116569] 00000008:00010000:0.0:1788201706.609574:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.609575:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.609581:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609590:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477466624 477470719) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609595:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609600:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609604:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.609608:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477466624, 477470720) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.609611:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116569 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.609614:0:17463:0:(rw26.c:898:ll_write_end()) pos 477466624, len 4096, copied 4096 00000080:00200000:0.0:1788201706.609616:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a1a4ae2e[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116569] 00000080:00200000:0.0:1788201706.609618:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c943b258 fffffc0001029 2:0 ffff93c0c12dab60 116569 lru 00000080:00200000:0.0:1788201706.609621:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007dda3f43 116569: 1< 2 - - > 2< 477466624 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68154 - -> 00000080:00200000:0.0:1788201706.609623:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a1a4ae2e 00000080:00200000:0.0:1788201706.609623:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.609625:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.609630:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.609638:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.609640:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.609641:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.609647:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.609649:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.609650:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.609653:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609658:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.609662:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.609664:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.609666:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.609668:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116569, 116569] 00000080:00200000:0.0:1788201706.609675:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.609677:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.609694:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.609697:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.609707:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477470720, count: 4096 00000080:00200000:0.0:1788201706.609713:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477470720, bytes: 4096 00000080:00200000:0.0:1788201706.609714:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.609717:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.609718:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477470720, 477474816) 0 0 00000080:00200000:0.0:1788201706.609719:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.609722:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116570, 116570] 00020000:00200000:0.0:1788201706.609724:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477470720 chunk: [477470720, 477474816) 477474816, 4096 00020000:00200000:0.0:1788201706.609726:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.609732:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116570, 116570] 00000008:00010000:0.0:1788201706.609735:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.609736:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.609741:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609747:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477470720 477474815) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609752:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609757:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609760:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.609765:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477470720, 477474816) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.609768:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116570 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.609772:0:17463:0:(rw26.c:898:ll_write_end()) pos 477470720, len 4096, copied 4096 00000080:00200000:0.0:1788201706.609774:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000802abba[3 0000000075f4bf38 2 1 00000000a9a2ef08 0 0 116570] 00000080:00200000:0.0:1788201706.609775:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e3310fa7 fffffc0001029 2:0 ffff93c0c12dac40 116570 lru 00000080:00200000:0.0:1788201706.609779:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005b932102 116570: 1< 2 - - > 2< 477470720 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68155 - -> 00000080:00200000:0.0:1788201706.609780:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000802abba 00000080:00200000:0.0:1788201706.609781:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.609782:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.609787:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.609795:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.609797:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.609798:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.609804:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.609805:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.609807:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.609809:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.609814:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.609818:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02998 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.609820:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.609822:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.609824:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116570, 116570] 00000080:00200000:0.0:1788201706.609847:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.609850:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.609882:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.609888:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.609910:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477474816, count: 4096 00000080:00200000:1.0:1788201706.609930:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477474816, bytes: 4096 00000080:00200000:1.0:1788201706.609933:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.609938:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.609940:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477474816, 477478912) 0 0 00000080:00200000:1.0:1788201706.609943:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.609949:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116571, 116571] 00020000:00200000:1.0:1788201706.609953:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477474816 chunk: [477474816, 477478912) 477478912, 4096 00020000:00200000:1.0:1788201706.609955:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.609970:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116571, 116571] 00000008:00010000:1.0:1788201706.609976:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.609978:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.609985:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.609993:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477474816 477478911) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.609999:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610005:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610012:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.610032:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477474816, 477478912) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.610038:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116571 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.610045:0:17463:0:(rw26.c:898:ll_write_end()) pos 477474816, len 4096, copied 4096 00000080:00200000:1.0:1788201706.610050:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002fd65143[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116571] 00000080:00200000:1.0:1788201706.610053:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003ad2c2ce fffffc0001029 2:0 ffff93c0c12da540 116571 lru 00000080:00200000:1.0:1788201706.610060:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bc5a0b21 116571: 1< 2 - - > 2< 477474816 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68156 - -> 00000080:00200000:1.0:1788201706.610062:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002fd65143 00000080:00200000:1.0:1788201706.610063:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.610065:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.610074:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.610084:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.610086:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.610088:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.610099:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.610101:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.610109:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.610113:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610118:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.610124:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.610125:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.610129:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.610132:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116571, 116571] 00000080:00200000:1.0:1788201706.610143:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.610146:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.610168:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.610171:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.610184:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477478912, count: 4096 00000080:00200000:1.0:1788201706.610190:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477478912, bytes: 4096 00000080:00200000:1.0:1788201706.610192:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.610195:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.610196:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477478912, 477483008) 0 0 00000080:00200000:1.0:1788201706.610198:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.610204:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116572, 116572] 00020000:00200000:1.0:1788201706.610206:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477478912 chunk: [477478912, 477483008) 477483008, 4096 00020000:00200000:1.0:1788201706.610209:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.610221:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116572, 116572] 00000008:00010000:1.0:1788201706.610227:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.610229:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.610238:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610256:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477478912 477483007) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610266:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610274:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610281:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.610288:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477478912, 477483008) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.610295:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116572 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.610301:0:17463:0:(rw26.c:898:ll_write_end()) pos 477478912, len 4096, copied 4096 00000080:00200000:1.0:1788201706.610305:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bb269e6a[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116572] 00000080:00200000:1.0:1788201706.610308:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cb1f04c3 fffffc0001029 2:0 ffff93c0c12da460 116572 lru 00000080:00200000:1.0:1788201706.610314:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000faec6a45 116572: 1< 2 - - > 2< 477478912 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68157 - -> 00000080:00200000:1.0:1788201706.610317:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bb269e6a 00000080:00200000:1.0:1788201706.610318:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.610320:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.610330:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.610342:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.610345:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.610347:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.610358:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.610361:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.610364:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.610368:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610376:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.610381:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.610383:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.610386:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.610388:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116572, 116572] 00000080:00200000:1.0:1788201706.610397:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.610399:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.610420:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.610422:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.610446:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477483008, count: 4096 00000080:00200000:1.0:1788201706.610453:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477483008, bytes: 4096 00000080:00200000:1.0:1788201706.610455:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.610458:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.610459:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477483008, 477487104) 0 0 00000080:00200000:1.0:1788201706.610461:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.610464:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116573, 116573] 00020000:00200000:1.0:1788201706.610467:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477483008 chunk: [477483008, 477487104) 477487104, 4096 00020000:00200000:1.0:1788201706.610468:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.610475:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116573, 116573] 00000008:00010000:1.0:1788201706.610479:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.610481:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.610486:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610492:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477483008 477487103) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610498:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610502:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610506:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.610515:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477483008, 477487104) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.610519:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116573 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.610523:0:17463:0:(rw26.c:898:ll_write_end()) pos 477483008, len 4096, copied 4096 00000080:00200000:1.0:1788201706.610525:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d7234de7[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116573] 00000080:00200000:1.0:1788201706.610528:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000c1ed7eb fffffc0001029 2:0 ffff93c0c12da0e0 116573 lru 00000080:00200000:1.0:1788201706.610532:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d8fa5f45 116573: 1< 2 - - > 2< 477483008 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68158 - -> 00000080:00200000:1.0:1788201706.610533:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d7234de7 00000080:00200000:1.0:1788201706.610534:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.610536:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.610542:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.610550:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.610552:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.610553:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.610560:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.610561:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.610563:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.610565:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610570:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.610575:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.610577:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.610579:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.610581:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116573, 116573] 00000080:00200000:1.0:1788201706.610588:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.610590:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.610609:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.610611:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.610622:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477487104, count: 4096 00000080:00200000:1.0:1788201706.610628:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477487104, bytes: 4096 00000080:00200000:1.0:1788201706.610630:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.610632:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.610633:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477487104, 477491200) 0 0 00000080:00200000:1.0:1788201706.610635:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.610638:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116574, 116574] 00020000:00200000:1.0:1788201706.610640:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477487104 chunk: [477487104, 477491200) 477491200, 4096 00020000:00200000:1.0:1788201706.610641:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.610647:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116574, 116574] 00000008:00010000:1.0:1788201706.610651:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.610652:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.610658:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610663:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477487104 477491199) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610669:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610673:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610681:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.610685:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477487104, 477491200) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.610688:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116574 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.610692:0:17463:0:(rw26.c:898:ll_write_end()) pos 477487104, len 4096, copied 4096 00000080:00200000:1.0:1788201706.610694:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000084783e68[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116574] 00000080:00200000:1.0:1788201706.610696:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007ab7edad fffffc0001029 2:0 ffff93c0c12da700 116574 lru 00000080:00200000:1.0:1788201706.610700:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002fd4a1ea 116574: 1< 2 - - > 2< 477487104 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68159 - -> 00000080:00200000:1.0:1788201706.610701:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000084783e68 00000080:00200000:1.0:1788201706.610702:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.610703:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.610709:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.610717:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.610719:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.610720:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.610726:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.610728:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.610729:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.610731:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610736:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.610745:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.610746:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.610749:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.610751:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116574, 116574] 00000080:00200000:1.0:1788201706.610758:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.610760:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.610778:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.610780:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.610790:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477491200, count: 4096 00000080:00200000:1.0:1788201706.610796:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477491200, bytes: 4096 00000080:00200000:1.0:1788201706.610798:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.610800:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.610801:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477491200, 477495296) 0 0 00000080:00200000:1.0:1788201706.610803:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.610806:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116575, 116575] 00020000:00200000:1.0:1788201706.610808:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477491200 chunk: [477491200, 477495296) 477495296, 4096 00020000:00200000:1.0:1788201706.610809:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.610815:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116575, 116575] 00000008:00010000:1.0:1788201706.610818:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.610820:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.610825:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610844:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477491200 477495295) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610852:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610865:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610872:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.610880:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477491200, 477495296) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.610886:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116575 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.610892:0:17463:0:(rw26.c:898:ll_write_end()) pos 477491200, len 4096, copied 4096 00000080:00200000:1.0:1788201706.610896:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d544faa2[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116575] 00000080:00200000:1.0:1788201706.610898:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b430ff98 fffffc0001029 2:0 ffff93c0c12da000 116575 lru 00000080:00200000:1.0:1788201706.610905:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f1163b92 116575: 1< 2 - - > 2< 477491200 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68160 - -> 00000080:00200000:1.0:1788201706.610907:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d544faa2 00000080:00200000:1.0:1788201706.610908:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.610910:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.610919:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.610931:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.610934:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.610936:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.610946:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.610949:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.610952:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.610955:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.610969:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.610977:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.610980:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.610985:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.610987:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116575, 116575] 00000080:00200000:1.0:1788201706.610996:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.610999:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.611041:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.611045:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.611060:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477495296, count: 4096 00000080:00200000:1.0:1788201706.611068:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477495296, bytes: 4096 00000080:00200000:1.0:1788201706.611070:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.611073:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.611074:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477495296, 477499392) 0 0 00000080:00200000:1.0:1788201706.611076:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.611080:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116576, 116576] 00020000:00200000:1.0:1788201706.611082:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477495296 chunk: [477495296, 477499392) 477499392, 4096 00020000:00200000:1.0:1788201706.611084:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.611091:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116576, 116576] 00000008:00010000:1.0:1788201706.611095:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.611096:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.611102:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611108:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477495296 477499391) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611122:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611130:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611136:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.611145:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477495296, 477499392) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.611150:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116576 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.611156:0:17463:0:(rw26.c:898:ll_write_end()) pos 477495296, len 4096, copied 4096 00000080:00200000:1.0:1788201706.611160:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d7256e2c[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116576] 00000080:00200000:1.0:1788201706.611163:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000a2d7e1c fffffc0001029 2:0 ffff93c0c12daa80 116576 lru 00000080:00200000:1.0:1788201706.611168:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a5d3e53a 116576: 1< 2 - - > 2< 477495296 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68161 - -> 00000080:00200000:1.0:1788201706.611170:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d7256e2c 00000080:00200000:1.0:1788201706.611170:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.611172:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.611179:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.611188:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.611190:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.611191:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.611201:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.611203:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.611206:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.611210:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611218:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.611226:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.611229:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.611233:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.611236:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116576, 116576] 00000080:00200000:1.0:1788201706.611247:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.611250:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.611270:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.611273:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.611283:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477499392, count: 4096 00000080:00200000:1.0:1788201706.611290:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477499392, bytes: 4096 00000080:00200000:1.0:1788201706.611291:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.611294:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.611295:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477499392, 477503488) 0 0 00000080:00200000:1.0:1788201706.611297:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.611300:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116577, 116577] 00020000:00200000:1.0:1788201706.611302:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477499392 chunk: [477499392, 477503488) 477503488, 4096 00020000:00200000:1.0:1788201706.611304:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.611310:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116577, 116577] 00000008:00010000:1.0:1788201706.611314:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.611315:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.611321:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611330:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477499392 477503487) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611336:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611340:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611344:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.611349:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477499392, 477503488) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.611352:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116577 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.611356:0:17463:0:(rw26.c:898:ll_write_end()) pos 477499392, len 4096, copied 4096 00000080:00200000:1.0:1788201706.611359:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000195eff3c[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116577] 00000080:00200000:1.0:1788201706.611361:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006542352d fffffc0001029 2:0 ffff93c0c12da380 116577 lru 00000080:00200000:1.0:1788201706.611365:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b7f28b0a 116577: 1< 2 - - > 2< 477499392 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68162 - -> 00000080:00200000:1.0:1788201706.611367:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000195eff3c 00000080:00200000:1.0:1788201706.611367:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.611369:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.611375:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.611383:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.611384:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.611386:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.611393:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.611394:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.611396:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.611398:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611403:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.611408:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.611410:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.611412:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.611414:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116577, 116577] 00000080:00200000:1.0:1788201706.611421:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.611422:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.611440:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.611442:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.611453:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477503488, count: 4096 00000080:00200000:1.0:1788201706.611459:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477503488, bytes: 4096 00000080:00200000:1.0:1788201706.611461:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.611463:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.611464:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477503488, 477507584) 0 0 00000080:00200000:1.0:1788201706.611466:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.611469:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116578, 116578] 00020000:00200000:1.0:1788201706.611471:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477503488 chunk: [477503488, 477507584) 477507584, 4096 00020000:00200000:1.0:1788201706.611472:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.611479:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116578, 116578] 00000008:00010000:1.0:1788201706.611482:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.611483:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.611489:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611494:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477503488 477507583) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611500:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611504:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611508:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.611512:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477503488, 477507584) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.611515:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116578 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.611518:0:17463:0:(rw26.c:898:ll_write_end()) pos 477503488, len 4096, copied 4096 00000080:00200000:1.0:1788201706.611521:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e2052450[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116578] 00000080:00200000:1.0:1788201706.611523:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000002d64dd5 fffffc0001029 2:0 ffff93c0c12da8c0 116578 lru 00000080:00200000:1.0:1788201706.611526:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000138bb750 116578: 1< 2 - - > 2< 477503488 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68163 - -> 00000080:00200000:1.0:1788201706.611528:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e2052450 00000080:00200000:1.0:1788201706.611529:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.611530:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.611535:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.611543:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.611545:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.611546:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.611553:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.611554:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.611556:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.611558:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611563:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.611568:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.611569:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.611571:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.611573:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116578, 116578] 00000080:00200000:1.0:1788201706.611580:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.611582:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.611599:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.611601:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.611611:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477507584, count: 4096 00000080:00200000:1.0:1788201706.611617:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477507584, bytes: 4096 00000080:00200000:1.0:1788201706.611619:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.611621:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.611622:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477507584, 477511680) 0 0 00000080:00200000:1.0:1788201706.611624:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.611627:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116579, 116579] 00020000:00200000:1.0:1788201706.611629:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477507584 chunk: [477507584, 477511680) 477511680, 4096 00020000:00200000:1.0:1788201706.611630:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.611636:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116579, 116579] 00000008:00010000:1.0:1788201706.611640:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.611641:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.611646:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611652:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477507584 477511679) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611657:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611662:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611665:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.611669:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477507584, 477511680) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.611672:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116579 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.611676:0:17463:0:(rw26.c:898:ll_write_end()) pos 477507584, len 4096, copied 4096 00000080:00200000:1.0:1788201706.611678:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009fa48bd5[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116579] 00000080:00200000:1.0:1788201706.611680:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001ac6159e fffffc0001029 2:0 ffff93c0c12daee0 116579 lru 00000080:00200000:1.0:1788201706.611684:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000051ad1e2b 116579: 1< 2 - - > 2< 477507584 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68164 - -> 00000080:00200000:1.0:1788201706.611685:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009fa48bd5 00000080:00200000:1.0:1788201706.611686:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.611687:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.611693:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.611704:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.611706:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.611707:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.611713:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.611715:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.611716:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.611719:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611724:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.611729:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.611730:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.611732:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.611734:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116579, 116579] 00000080:00200000:1.0:1788201706.611740:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.611742:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.611760:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.611762:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.611774:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477511680, count: 4096 00000080:00200000:1.0:1788201706.611784:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477511680, bytes: 4096 00000080:00200000:1.0:1788201706.611787:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.611790:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.611792:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477511680, 477515776) 0 0 00000080:00200000:1.0:1788201706.611794:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.611799:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116580, 116580] 00020000:00200000:1.0:1788201706.611803:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477511680 chunk: [477511680, 477515776) 477515776, 4096 00020000:00200000:1.0:1788201706.611805:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.611817:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116580, 116580] 00000008:00010000:1.0:1788201706.611823:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.611825:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.611849:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611860:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477511680 477515775) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611866:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611871:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611876:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.611880:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477511680, 477515776) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.611884:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116580 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.611888:0:17463:0:(rw26.c:898:ll_write_end()) pos 477511680, len 4096, copied 4096 00000080:00200000:1.0:1788201706.611890:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001945841a[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116580] 00000080:00200000:1.0:1788201706.611892:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004756c5a2 fffffc0001029 2:0 ffff93c0c12da7e0 116580 lru 00000080:00200000:1.0:1788201706.611895:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006559145a 116580: 1< 2 - - > 2< 477511680 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68165 - -> 00000080:00200000:1.0:1788201706.611897:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001945841a 00000080:00200000:1.0:1788201706.611897:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.611899:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.611905:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.611916:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.611919:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.611921:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.611930:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.611933:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.611935:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.611939:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.611947:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.611955:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.611957:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.611962:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.611965:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116580, 116580] 00000080:00200000:1.0:1788201706.611976:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.611979:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.612002:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.612006:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.612034:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477515776, count: 4096 00000080:00200000:1.0:1788201706.612046:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477515776, bytes: 4096 00000080:00200000:1.0:1788201706.612049:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.612052:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.612054:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477515776, 477519872) 0 0 00000080:00200000:1.0:1788201706.612057:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.612062:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116581, 116581] 00020000:00200000:1.0:1788201706.612065:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477515776 chunk: [477515776, 477519872) 477519872, 4096 00020000:00200000:1.0:1788201706.612067:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.612076:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116581, 116581] 00000008:00010000:1.0:1788201706.612080:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.612081:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.612087:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612092:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477515776 477519871) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612098:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612103:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612107:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.612113:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477515776, 477519872) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.612118:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116581 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.612123:0:17463:0:(rw26.c:898:ll_write_end()) pos 477515776, len 4096, copied 4096 00000080:00200000:1.0:1788201706.612127:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000057fb3a0c[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116581] 00000080:00200000:1.0:1788201706.612130:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000060d66b6 fffffc0001029 2:0 ffff93c0c12da2a0 116581 lru 00000080:00200000:1.0:1788201706.612135:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000a139faf 116581: 1< 2 - - > 2< 477515776 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68166 - -> 00000080:00200000:1.0:1788201706.612139:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000057fb3a0c 00000080:00200000:1.0:1788201706.612140:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.612142:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.612151:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.612163:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.612166:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.612167:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.612175:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.612176:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.612178:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.612181:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612186:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.612191:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.612192:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.612196:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.612198:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116581, 116581] 00000080:00200000:1.0:1788201706.612209:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.612212:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.612240:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.612244:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.612256:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477519872, count: 4096 00000080:00200000:1.0:1788201706.612262:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477519872, bytes: 4096 00000080:00200000:1.0:1788201706.612265:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.612267:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.612269:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477519872, 477523968) 0 0 00000080:00200000:1.0:1788201706.612270:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.612274:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116582, 116582] 00020000:00200000:1.0:1788201706.612276:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477519872 chunk: [477519872, 477523968) 477523968, 4096 00020000:00200000:1.0:1788201706.612277:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.612284:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116582, 116582] 00000008:00010000:1.0:1788201706.612287:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.612289:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.612294:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612300:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477519872 477523967) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612306:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612310:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612314:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.612319:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477519872, 477523968) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.612322:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116582 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.612326:0:17463:0:(rw26.c:898:ll_write_end()) pos 477519872, len 4096, copied 4096 00000080:00200000:1.0:1788201706.612328:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e4eebc0f[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116582] 00000080:00200000:1.0:1788201706.612331:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000320fe57c fffffc0001029 2:0 ffff93c0c12dae00 116582 lru 00000080:00200000:1.0:1788201706.612335:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000d2588e6 116582: 1< 2 - - > 2< 477519872 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68167 - -> 00000080:00200000:1.0:1788201706.612336:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e4eebc0f 00000080:00200000:1.0:1788201706.612337:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.612338:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.612344:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.612351:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.612353:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.612354:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.612361:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.612362:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.612364:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.612366:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612371:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.612376:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.612378:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.612380:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.612382:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116582, 116582] 00000080:00200000:1.0:1788201706.612389:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.612391:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.612409:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.612411:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.612422:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477523968, count: 4096 00000080:00200000:1.0:1788201706.612427:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477523968, bytes: 4096 00000080:00200000:1.0:1788201706.612429:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.612431:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.612432:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477523968, 477528064) 0 0 00000080:00200000:1.0:1788201706.612434:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.612437:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116583, 116583] 00020000:00200000:1.0:1788201706.612439:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477523968 chunk: [477523968, 477528064) 477528064, 4096 00020000:00200000:1.0:1788201706.612440:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.612447:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116583, 116583] 00000008:00010000:1.0:1788201706.612450:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.612451:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.612456:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612461:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477523968 477528063) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612467:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612471:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612475:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.612479:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477523968, 477528064) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.612482:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116583 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.612486:0:17463:0:(rw26.c:898:ll_write_end()) pos 477523968, len 4096, copied 4096 00000080:00200000:1.0:1788201706.612488:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a08b9277[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116583] 00000080:00200000:1.0:1788201706.612490:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d6d8f448 fffffc0001029 2:0 ffff93c0c12da1c0 116583 lru 00000080:00200000:1.0:1788201706.612494:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000be03c6e3 116583: 1< 2 - - > 2< 477523968 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68168 - -> 00000080:00200000:1.0:1788201706.612495:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a08b9277 00000080:00200000:1.0:1788201706.612496:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.612497:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.612503:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.612510:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.612512:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.612513:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.612519:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.612521:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.612522:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.612525:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612530:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.612534:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.612536:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.612538:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.612540:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116583, 116583] 00000080:00200000:1.0:1788201706.612546:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.612548:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.612565:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.612568:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.612577:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477528064, count: 4096 00000080:00200000:1.0:1788201706.612583:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477528064, bytes: 4096 00000080:00200000:1.0:1788201706.612585:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.612587:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.612588:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477528064, 477532160) 0 0 00000080:00200000:1.0:1788201706.612590:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.612593:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116584, 116584] 00020000:00200000:1.0:1788201706.612595:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477528064 chunk: [477528064, 477532160) 477532160, 4096 00020000:00200000:1.0:1788201706.612596:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.612602:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116584, 116584] 00000008:00010000:1.0:1788201706.612605:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.612607:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.612611:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612617:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477528064 477532159) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612623:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612627:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612631:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.612635:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477528064, 477532160) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.612638:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116584 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.612642:0:17463:0:(rw26.c:898:ll_write_end()) pos 477528064, len 4096, copied 4096 00000080:00200000:1.0:1788201706.612644:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f48cc0db[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116584] 00000080:00200000:1.0:1788201706.612646:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000036e7b303 fffffc0001029 2:0 ffff93c0c12db460 116584 lru 00000080:00200000:1.0:1788201706.612650:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000079f92436 116584: 1< 2 - - > 2< 477528064 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68169 - -> 00000080:00200000:1.0:1788201706.612651:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f48cc0db 00000080:00200000:1.0:1788201706.612652:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.612653:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.612659:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.612666:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.612668:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.612669:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.612675:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.612677:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.612678:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.612681:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612685:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.612690:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.612691:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.612694:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.612696:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116584, 116584] 00000080:00200000:1.0:1788201706.612703:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.612705:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.612722:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.612724:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.612734:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477532160, count: 4096 00000080:00200000:1.0:1788201706.612740:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477532160, bytes: 4096 00000080:00200000:1.0:1788201706.612742:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.612744:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.612745:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477532160, 477536256) 0 0 00000080:00200000:1.0:1788201706.612747:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.612750:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116585, 116585] 00020000:00200000:1.0:1788201706.612752:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477532160 chunk: [477532160, 477536256) 477536256, 4096 00020000:00200000:1.0:1788201706.612753:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.612759:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116585, 116585] 00000008:00010000:1.0:1788201706.612762:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.612763:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.612768:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612774:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477532160 477536255) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612779:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612784:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612787:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.612795:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477532160, 477536256) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.612798:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116585 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.612802:0:17463:0:(rw26.c:898:ll_write_end()) pos 477532160, len 4096, copied 4096 00000080:00200000:1.0:1788201706.612804:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007d89be49[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116585] 00000080:00200000:1.0:1788201706.612806:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000432374e9 fffffc0001029 2:0 ffff93c0c12db0e0 116585 lru 00000080:00200000:1.0:1788201706.612810:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005f836b2f 116585: 1< 2 - - > 2< 477532160 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68170 - -> 00000080:00200000:1.0:1788201706.612811:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007d89be49 00000080:00200000:1.0:1788201706.612812:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.612813:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.612819:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.612826:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.612828:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.612843:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.612849:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.612851:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.612852:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.612855:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612860:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.612865:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.612866:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.612869:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.612871:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116585, 116585] 00000080:00200000:1.0:1788201706.612878:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.612880:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.612897:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.612899:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.612909:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477536256, count: 4096 00000080:00200000:1.0:1788201706.612918:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477536256, bytes: 4096 00000080:00200000:1.0:1788201706.612920:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.612922:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.612923:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477536256, 477540352) 0 0 00000080:00200000:1.0:1788201706.612925:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.612928:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116586, 116586] 00020000:00200000:1.0:1788201706.612930:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477536256 chunk: [477536256, 477540352) 477540352, 4096 00020000:00200000:1.0:1788201706.612932:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.612939:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116586, 116586] 00000008:00010000:1.0:1788201706.612942:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.612943:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.612948:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612954:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477536256 477540351) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612959:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612964:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.612971:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.612976:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477536256, 477540352) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.612979:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116586 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.612982:0:17463:0:(rw26.c:898:ll_write_end()) pos 477536256, len 4096, copied 4096 00000080:00200000:1.0:1788201706.612985:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f5d86249[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116586] 00000080:00200000:1.0:1788201706.612986:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000096afd0b7 fffffc0001029 2:0 ffff93c0c12db700 116586 lru 00000080:00200000:1.0:1788201706.612990:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000059195ab9 116586: 1< 2 - - > 2< 477536256 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68171 - -> 00000080:00200000:1.0:1788201706.612992:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f5d86249 00000080:00200000:1.0:1788201706.612992:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.612994:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.613000:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.613007:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.613009:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.613010:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.613029:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.613032:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.613033:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.613036:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613041:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.613050:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.613051:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.613054:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.613056:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116586, 116586] 00000080:00200000:1.0:1788201706.613063:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.613065:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.613084:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.613086:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.613096:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477540352, count: 4096 00000080:00200000:1.0:1788201706.613102:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477540352, bytes: 4096 00000080:00200000:1.0:1788201706.613104:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.613106:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.613107:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477540352, 477544448) 0 0 00000080:00200000:1.0:1788201706.613109:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.613112:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116587, 116587] 00020000:00200000:1.0:1788201706.613114:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477540352 chunk: [477540352, 477544448) 477544448, 4096 00020000:00200000:1.0:1788201706.613116:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.613122:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116587, 116587] 00000008:00010000:1.0:1788201706.613125:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.613126:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.613131:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613137:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477540352 477544447) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613143:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613151:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613155:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.613159:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477540352, 477544448) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.613162:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116587 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.613166:0:17463:0:(rw26.c:898:ll_write_end()) pos 477540352, len 4096, copied 4096 00000080:00200000:1.0:1788201706.613168:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000095c13ede[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116587] 00000080:00200000:1.0:1788201706.613169:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001fc6cfb4 fffffc0001029 2:0 ffff93c0c12db000 116587 lru 00000080:00200000:1.0:1788201706.613173:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bac5f622 116587: 1< 2 - - > 2< 477540352 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68172 - -> 00000080:00200000:1.0:1788201706.613174:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000095c13ede 00000080:00200000:1.0:1788201706.613175:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.613176:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.613182:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.613189:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.613191:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.613192:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.613200:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.613202:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.613205:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.613209:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613221:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.613230:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.613232:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.613237:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.613240:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116587, 116587] 00000080:00200000:1.0:1788201706.613249:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.613251:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.613270:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.613272:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.613283:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477544448, count: 4096 00000080:00200000:1.0:1788201706.613289:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477544448, bytes: 4096 00000080:00200000:1.0:1788201706.613290:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.613293:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.613294:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477544448, 477548544) 0 0 00000080:00200000:1.0:1788201706.613295:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.613298:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116588, 116588] 00020000:00200000:1.0:1788201706.613300:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477544448 chunk: [477544448, 477548544) 477548544, 4096 00020000:00200000:1.0:1788201706.613302:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.613308:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116588, 116588] 00000008:00010000:1.0:1788201706.613312:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.613313:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.613318:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613324:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477544448 477548543) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613333:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613338:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613342:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.613346:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477544448, 477548544) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.613349:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116588 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.613353:0:17463:0:(rw26.c:898:ll_write_end()) pos 477544448, len 4096, copied 4096 00000080:00200000:1.0:1788201706.613355:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d2c5c59c[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116588] 00000080:00200000:1.0:1788201706.613357:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c3abcaef fffffc0001029 2:0 ffff93c0c12dba80 116588 lru 00000080:00200000:1.0:1788201706.613360:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e595a133 116588: 1< 2 - - > 2< 477544448 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68173 - -> 00000080:00200000:1.0:1788201706.613361:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d2c5c59c 00000080:00200000:1.0:1788201706.613362:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.613363:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.613369:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.613377:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.613379:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.613380:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.613386:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.613388:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.613390:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.613392:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613397:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.613402:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.613403:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.613406:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.613408:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116588, 116588] 00000080:00200000:1.0:1788201706.613414:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.613416:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.613434:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.613436:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.613446:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477548544, count: 4096 00000080:00200000:1.0:1788201706.613452:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477548544, bytes: 4096 00000080:00200000:1.0:1788201706.613454:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.613456:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.613457:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477548544, 477552640) 0 0 00000080:00200000:1.0:1788201706.613458:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.613462:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116589, 116589] 00020000:00200000:1.0:1788201706.613463:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477548544 chunk: [477548544, 477552640) 477552640, 4096 00020000:00200000:1.0:1788201706.613465:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.613471:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116589, 116589] 00000008:00010000:1.0:1788201706.613474:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.613476:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.613480:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613489:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477548544 477552639) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613495:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613499:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613503:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.613507:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477548544, 477552640) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.613510:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116589 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.613514:0:17463:0:(rw26.c:898:ll_write_end()) pos 477548544, len 4096, copied 4096 00000080:00200000:1.0:1788201706.613516:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000be6f0ff0[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116589] 00000080:00200000:1.0:1788201706.613518:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c563385f fffffc0001029 2:0 ffff93c0c12db380 116589 lru 00000080:00200000:1.0:1788201706.613522:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000422df446 116589: 1< 2 - - > 2< 477548544 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68174 - -> 00000080:00200000:1.0:1788201706.613523:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000be6f0ff0 00000080:00200000:1.0:1788201706.613524:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.613525:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.613530:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.613538:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.613540:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.613541:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.613547:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.613549:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.613550:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.613553:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613558:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.613562:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.613564:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.613566:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.613568:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116589, 116589] 00000080:00200000:1.0:1788201706.613574:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.613576:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.613593:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.613595:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.613605:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477552640, count: 4096 00000080:00200000:1.0:1788201706.613611:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477552640, bytes: 4096 00000080:00200000:1.0:1788201706.613613:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.613615:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.613617:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477552640, 477556736) 0 0 00000080:00200000:1.0:1788201706.613618:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.613621:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116590, 116590] 00020000:00200000:1.0:1788201706.613623:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477552640 chunk: [477552640, 477556736) 477556736, 4096 00020000:00200000:1.0:1788201706.613625:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.613631:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116590, 116590] 00000008:00010000:1.0:1788201706.613634:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.613635:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.613640:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613646:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477552640 477556735) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613651:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613656:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613659:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.613663:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477552640, 477556736) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.613666:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116590 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.613670:0:17463:0:(rw26.c:898:ll_write_end()) pos 477552640, len 4096, copied 4096 00000080:00200000:1.0:1788201706.613672:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fed21d97[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116590] 00000080:00200000:1.0:1788201706.613674:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bae6e6fc fffffc0001029 2:0 ffff93c0c12db8c0 116590 lru 00000080:00200000:1.0:1788201706.613678:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000064920915 116590: 1< 2 - - > 2< 477552640 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68175 - -> 00000080:00200000:1.0:1788201706.613679:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fed21d97 00000080:00200000:1.0:1788201706.613680:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.613681:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.613686:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.613693:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.613695:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.613697:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.613702:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.613704:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.613705:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.613708:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613713:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.613718:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.613719:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.613721:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.613723:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116590, 116590] 00000080:00200000:1.0:1788201706.613729:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.613731:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.613748:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.613751:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.613761:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477556736, count: 4096 00000080:00200000:1.0:1788201706.613767:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477556736, bytes: 4096 00000080:00200000:1.0:1788201706.613768:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.613770:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.613771:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477556736, 477560832) 0 0 00000080:00200000:1.0:1788201706.613773:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.613776:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116591, 116591] 00020000:00200000:1.0:1788201706.613778:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477556736 chunk: [477556736, 477560832) 477560832, 4096 00020000:00200000:1.0:1788201706.613779:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.613785:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116591, 116591] 00000008:00010000:1.0:1788201706.613789:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.613790:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.613795:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613801:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477556736 477560831) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613806:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613810:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613814:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.613818:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477556736, 477560832) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.613821:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116591 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.613824:0:17463:0:(rw26.c:898:ll_write_end()) pos 477556736, len 4096, copied 4096 00000080:00200000:1.0:1788201706.613826:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000006b9a59[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116591] 00000080:00200000:1.0:1788201706.613828:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004c1067cb fffffc0001029 2:0 ffff93c0c12dbee0 116591 lru 00000080:00200000:1.0:1788201706.613845:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000060284b5b 116591: 1< 2 - - > 2< 477556736 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68176 - -> 00000080:00200000:1.0:1788201706.613846:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000006b9a59 00000080:00200000:1.0:1788201706.613846:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.613848:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.613854:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.613864:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.613866:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.613867:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.613873:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.613875:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.613877:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.613879:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613884:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.613889:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.613890:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.613892:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.613894:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116591, 116591] 00000080:00200000:1.0:1788201706.613901:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.613903:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.613920:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.613923:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.613932:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477560832, count: 4096 00000080:00200000:1.0:1788201706.613938:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477560832, bytes: 4096 00000080:00200000:1.0:1788201706.613940:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.613942:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.613943:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477560832, 477564928) 0 0 00000080:00200000:1.0:1788201706.613945:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.613948:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116592, 116592] 00020000:00200000:1.0:1788201706.613950:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477560832 chunk: [477560832, 477564928) 477564928, 4096 00020000:00200000:1.0:1788201706.613951:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.613958:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116592, 116592] 00000008:00010000:1.0:1788201706.613961:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.613962:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.613967:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613972:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477560832 477564927) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613978:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613982:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.613986:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.613990:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477560832, 477564928) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.613993:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116592 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.613998:0:17463:0:(rw26.c:898:ll_write_end()) pos 477560832, len 4096, copied 4096 00000080:00200000:1.0:1788201706.614000:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000095031fd5[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116592] 00000080:00200000:1.0:1788201706.614002:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a5a0e401 fffffc0001029 2:0 ffff93c0c12db7e0 116592 lru 00000080:00200000:1.0:1788201706.614006:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003f60fee9 116592: 1< 2 - - > 2< 477560832 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68177 - -> 00000080:00200000:1.0:1788201706.614007:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000095031fd5 00000080:00200000:1.0:1788201706.614008:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.614010:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.614028:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.614038:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.614040:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.614041:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.614048:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.614050:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.614051:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.614054:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614059:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.614064:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.614065:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.614067:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.614069:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116592, 116592] 00000080:00200000:1.0:1788201706.614076:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.614078:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.614096:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.614098:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.614108:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477564928, count: 4096 00000080:00200000:1.0:1788201706.614114:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477564928, bytes: 4096 00000080:00200000:1.0:1788201706.614116:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.614118:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.614119:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477564928, 477569024) 0 0 00000080:00200000:1.0:1788201706.614121:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.614124:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116593, 116593] 00020000:00200000:1.0:1788201706.614126:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477564928 chunk: [477564928, 477569024) 477569024, 4096 00020000:00200000:1.0:1788201706.614127:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.614134:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116593, 116593] 00000008:00010000:1.0:1788201706.614137:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.614138:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.614143:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614149:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477564928 477569023) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614155:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614159:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614163:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.614168:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477564928, 477569024) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.614171:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116593 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.614175:0:17463:0:(rw26.c:898:ll_write_end()) pos 477564928, len 4096, copied 4096 00000080:00200000:1.0:1788201706.614177:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000041237d7f[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116593] 00000080:00200000:1.0:1788201706.614179:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006e27398e fffffc0001029 2:0 ffff93c0c12db2a0 116593 lru 00000080:00200000:1.0:1788201706.614183:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005b009bdd 116593: 1< 2 - - > 2< 477564928 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68178 - -> 00000080:00200000:1.0:1788201706.614185:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000041237d7f 00000080:00200000:1.0:1788201706.614186:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.614187:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.614192:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.614203:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.614206:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.614208:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.614216:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.614219:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.614221:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.614225:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614234:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.614242:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.614244:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.614248:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.614250:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116593, 116593] 00000080:00200000:1.0:1788201706.614258:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.614260:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.614278:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.614280:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.614291:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477569024, count: 4096 00000080:00200000:1.0:1788201706.614296:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477569024, bytes: 4096 00000080:00200000:1.0:1788201706.614298:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.614301:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.614302:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477569024, 477573120) 0 0 00000080:00200000:1.0:1788201706.614303:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.614306:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116594, 116594] 00020000:00200000:1.0:1788201706.614308:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477569024 chunk: [477569024, 477573120) 477573120, 4096 00020000:00200000:1.0:1788201706.614310:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.614316:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116594, 116594] 00000008:00010000:1.0:1788201706.614319:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.614321:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.614326:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614332:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477569024 477573119) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614337:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614342:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614345:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.614350:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477569024, 477573120) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.614353:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116594 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.614357:0:17463:0:(rw26.c:898:ll_write_end()) pos 477569024, len 4096, copied 4096 00000080:00200000:1.0:1788201706.614359:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007d238fe4[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116594] 00000080:00200000:1.0:1788201706.614362:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000c0a7537 fffffc0001029 2:0 ffff93c0c12dbe00 116594 lru 00000080:00200000:1.0:1788201706.614365:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cda4b868 116594: 1< 2 - - > 2< 477569024 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68179 - -> 00000080:00200000:1.0:1788201706.614367:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007d238fe4 00000080:00200000:1.0:1788201706.614367:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.614369:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.614374:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.614381:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.614383:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.614384:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.614390:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.614392:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.614393:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.614396:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614401:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.614405:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.614407:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.614409:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.614411:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116594, 116594] 00000080:00200000:1.0:1788201706.614417:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.614419:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.614436:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.614439:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.614449:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477573120, count: 4096 00000080:00200000:1.0:1788201706.614455:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477573120, bytes: 4096 00000080:00200000:1.0:1788201706.614456:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.614459:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.614460:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477573120, 477577216) 0 0 00000080:00200000:1.0:1788201706.614461:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.614464:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116595, 116595] 00020000:00200000:1.0:1788201706.614466:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477573120 chunk: [477573120, 477577216) 477577216, 4096 00020000:00200000:1.0:1788201706.614467:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.614474:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116595, 116595] 00000008:00010000:1.0:1788201706.614477:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.614478:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.614483:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614489:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477573120 477577215) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614494:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614499:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614503:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.614507:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477573120, 477577216) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.614510:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116595 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.614514:0:17463:0:(rw26.c:898:ll_write_end()) pos 477573120, len 4096, copied 4096 00000080:00200000:1.0:1788201706.614517:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005a9f3fe5[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116595] 00000080:00200000:1.0:1788201706.614518:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000008241f52 fffffc0001029 2:0 ffff93c0c12db1c0 116595 lru 00000080:00200000:1.0:1788201706.614522:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000098768829 116595: 1< 2 - - > 2< 477573120 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68180 - -> 00000080:00200000:1.0:1788201706.614523:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005a9f3fe5 00000080:00200000:1.0:1788201706.614524:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.614525:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.614531:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.614538:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.614540:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.614541:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.614547:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.614549:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.614550:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.614553:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614557:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.614562:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.614564:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.614566:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.614568:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116595, 116595] 00000080:00200000:1.0:1788201706.614575:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.614577:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.614594:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.614597:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.614606:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477577216, count: 4096 00000080:00200000:1.0:1788201706.614612:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477577216, bytes: 4096 00000080:00200000:1.0:1788201706.614614:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.614617:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.614618:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477577216, 477581312) 0 0 00000080:00200000:1.0:1788201706.614619:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.614622:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116596, 116596] 00020000:00200000:1.0:1788201706.614624:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477577216 chunk: [477577216, 477581312) 477581312, 4096 00020000:00200000:1.0:1788201706.614625:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.614632:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116596, 116596] 00000008:00010000:1.0:1788201706.614635:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.614636:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.614641:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614646:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477577216 477581311) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614652:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614656:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614660:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.614665:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477577216, 477581312) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.614668:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116596 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.614672:0:17463:0:(rw26.c:898:ll_write_end()) pos 477577216, len 4096, copied 4096 00000080:00200000:1.0:1788201706.614674:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003e0d3f9e[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116596] 00000080:00200000:1.0:1788201706.614676:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000054493c2f fffffc0001029 2:0 ffff93c0c12db620 116596 lru 00000080:00200000:1.0:1788201706.614680:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000270c1461 116596: 1< 2 - - > 2< 477577216 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68181 - -> 00000080:00200000:1.0:1788201706.614681:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003e0d3f9e 00000080:00200000:1.0:1788201706.614682:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.614683:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.614689:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.614696:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.614698:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.614699:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.614705:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.614706:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.614708:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.614710:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614714:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.614719:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.614720:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.614723:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.614725:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116596, 116596] 00000080:00200000:1.0:1788201706.614731:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.614733:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.614751:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.614753:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.614763:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477581312, count: 4096 00000080:00200000:1.0:1788201706.614769:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477581312, bytes: 4096 00000080:00200000:1.0:1788201706.614770:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.614773:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.614774:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477581312, 477585408) 0 0 00000080:00200000:1.0:1788201706.614776:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.614779:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116597, 116597] 00020000:00200000:1.0:1788201706.614780:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477581312 chunk: [477581312, 477585408) 477585408, 4096 00020000:00200000:1.0:1788201706.614782:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.614788:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116597, 116597] 00000008:00010000:1.0:1788201706.614791:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.614792:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.614797:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614803:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477581312 477585407) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614808:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614812:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614816:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.614824:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477581312, 477585408) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.614827:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116597 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.614843:0:17463:0:(rw26.c:898:ll_write_end()) pos 477581312, len 4096, copied 4096 00000080:00200000:1.0:1788201706.614846:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b4e5e4f5[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116597] 00000080:00200000:1.0:1788201706.614848:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e928e021 fffffc0001029 2:0 ffff93c0c12dbd20 116597 lru 00000080:00200000:1.0:1788201706.614852:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a2bb17ae 116597: 1< 2 - - > 2< 477581312 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68182 - -> 00000080:00200000:1.0:1788201706.614853:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b4e5e4f5 00000080:00200000:1.0:1788201706.614854:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.614855:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.614861:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.614868:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.614870:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.614871:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.614877:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.614879:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.614880:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.614883:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.614888:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.614893:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.614894:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.614897:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.614899:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116597, 116597] 00000080:00200000:1.0:1788201706.614905:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.614907:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.614931:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.614934:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.614948:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477585408, count: 4096 00000080:00200000:1.0:1788201706.614958:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477585408, bytes: 4096 00000080:00200000:1.0:1788201706.614961:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.614965:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.614966:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477585408, 477589504) 0 0 00000080:00200000:1.0:1788201706.614968:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.614972:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116598, 116598] 00020000:00200000:1.0:1788201706.614975:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477585408 chunk: [477585408, 477589504) 477589504, 4096 00020000:00200000:1.0:1788201706.614977:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.614987:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116598, 116598] 00000008:00010000:1.0:1788201706.614992:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.614994:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.615001:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615009:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477585408 477589503) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615030:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615037:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615047:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.615055:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477585408, 477589504) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.615060:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116598 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.615065:0:17463:0:(rw26.c:898:ll_write_end()) pos 477585408, len 4096, copied 4096 00000080:00200000:1.0:1788201706.615069:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c59b25c4[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116598] 00000080:00200000:1.0:1788201706.615072:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000096a77e6 fffffc0001029 2:0 ffff93c0c12db9a0 116598 lru 00000080:00200000:1.0:1788201706.615077:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000daeea03c 116598: 1< 2 - - > 2< 477585408 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68183 - -> 00000080:00200000:1.0:1788201706.615079:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c59b25c4 00000080:00200000:1.0:1788201706.615080:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.615081:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.615089:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.615100:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.615103:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.615104:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.615113:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.615116:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.615118:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.615123:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615131:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.615144:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.615147:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.615150:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.615153:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116598, 116598] 00000080:00200000:1.0:1788201706.615164:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.615167:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.615193:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.615197:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.615212:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477589504, count: 4096 00000080:00200000:1.0:1788201706.615222:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477589504, bytes: 4096 00000080:00200000:1.0:1788201706.615225:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.615228:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.615229:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477589504, 477593600) 0 0 00000080:00200000:1.0:1788201706.615231:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.615236:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116599, 116599] 00020000:00200000:1.0:1788201706.615239:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477589504 chunk: [477589504, 477593600) 477593600, 4096 00020000:00200000:1.0:1788201706.615241:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.615253:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116599, 116599] 00000008:00010000:1.0:1788201706.615258:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.615261:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.615269:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615277:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477589504 477593599) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615286:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615298:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615305:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.615312:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477589504, 477593600) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.615318:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116599 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.615322:0:17463:0:(rw26.c:898:ll_write_end()) pos 477589504, len 4096, copied 4096 00000080:00200000:1.0:1788201706.615324:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000907b0c5b[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116599] 00000080:00200000:1.0:1788201706.615326:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003d58c231 fffffc0001029 2:0 ffff93c0c12dbb60 116599 lru 00000080:00200000:1.0:1788201706.615330:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ab03fcc2 116599: 1< 2 - - > 2< 477589504 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68184 - -> 00000080:00200000:1.0:1788201706.615332:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000907b0c5b 00000080:00200000:1.0:1788201706.615332:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.615334:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.615340:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.615348:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.615349:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.615351:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.615357:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.615359:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.615361:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.615363:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615372:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.615377:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.615378:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.615380:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.615382:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116599, 116599] 00000080:00200000:1.0:1788201706.615392:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.615395:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.615424:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.615427:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.615443:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477593600, count: 4096 00000080:00200000:1.0:1788201706.615454:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477593600, bytes: 4096 00000080:00200000:1.0:1788201706.615457:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.615461:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.615462:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477593600, 477597696) 0 0 00000080:00200000:1.0:1788201706.615464:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.615470:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116600, 116600] 00020000:00200000:1.0:1788201706.615473:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477593600 chunk: [477593600, 477597696) 477597696, 4096 00020000:00200000:1.0:1788201706.615475:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.615487:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116600, 116600] 00000008:00010000:1.0:1788201706.615493:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.615495:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.615503:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615512:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477593600 477597695) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615527:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615535:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615540:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.615545:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477593600, 477597696) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.615548:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116600 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.615552:0:17463:0:(rw26.c:898:ll_write_end()) pos 477593600, len 4096, copied 4096 00000080:00200000:1.0:1788201706.615555:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e010a10a[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116600] 00000080:00200000:1.0:1788201706.615557:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000991a8f59 fffffc0001029 2:0 ffff93c0c12dbc40 116600 lru 00000080:00200000:1.0:1788201706.615560:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009a9d2b58 116600: 1< 2 - - > 2< 477593600 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68185 - -> 00000080:00200000:1.0:1788201706.615562:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e010a10a 00000080:00200000:1.0:1788201706.615563:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.615564:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.615572:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.615584:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.615587:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.615589:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.615599:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.615601:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.615604:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.615608:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615616:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.615626:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.615628:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.615632:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.615635:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116600, 116600] 00000080:00200000:1.0:1788201706.615647:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.615650:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.615679:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.615683:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.615699:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477597696, count: 4096 00000080:00200000:1.0:1788201706.615711:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477597696, bytes: 4096 00000080:00200000:1.0:1788201706.615713:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.615717:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.615719:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477597696, 477601792) 0 0 00000080:00200000:1.0:1788201706.615721:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.615726:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116601, 116601] 00020000:00200000:1.0:1788201706.615729:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477597696 chunk: [477597696, 477601792) 477601792, 4096 00020000:00200000:1.0:1788201706.615731:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.615743:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116601, 116601] 00000008:00010000:1.0:1788201706.615749:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.615751:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.615759:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615773:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477597696 477601791) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615782:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615790:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615797:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.615805:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477597696, 477601792) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.615810:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116601 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.615815:0:17463:0:(rw26.c:898:ll_write_end()) pos 477597696, len 4096, copied 4096 00000080:00200000:1.0:1788201706.615820:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000db741414[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116601] 00000080:00200000:1.0:1788201706.615823:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000160e0ca5 fffffc0001029 2:0 ffff93c0c12db540 116601 lru 00000080:00200000:1.0:1788201706.615846:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000170aca6c 116601: 1< 2 - - > 2< 477597696 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68186 - -> 00000080:00200000:1.0:1788201706.615849:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000db741414 00000080:00200000:1.0:1788201706.615850:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.615852:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.615862:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.615875:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.615878:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.615880:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.615889:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.615893:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.615895:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.615899:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.615908:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.615916:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.615918:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.615922:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.615925:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116601, 116601] 00000080:00200000:1.0:1788201706.615936:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.615939:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.615969:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.615973:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.615989:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477601792, count: 4096 00000080:00200000:1.0:1788201706.616000:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477601792, bytes: 4096 00000080:00200000:1.0:1788201706.616003:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.616007:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.616009:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477601792, 477605888) 0 0 00000080:00200000:1.0:1788201706.616011:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.616029:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116602, 116602] 00020000:00200000:1.0:1788201706.616033:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477601792 chunk: [477601792, 477605888) 477605888, 4096 00020000:00200000:1.0:1788201706.616035:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.616049:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116602, 116602] 00000008:00010000:1.0:1788201706.616054:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.616057:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.616065:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616074:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477601792 477605887) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616083:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616091:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616099:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.616107:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477601792, 477605888) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.616112:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116602 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.616117:0:17463:0:(rw26.c:898:ll_write_end()) pos 477601792, len 4096, copied 4096 00000080:00200000:1.0:1788201706.616122:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001e2c63e6[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116602] 00000080:00200000:1.0:1788201706.616125:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001b2ed1e5 fffffc0001029 2:0 ffff93c0c12dc380 116602 lru 00000080:00200000:1.0:1788201706.616131:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000898168c5 116602: 1< 2 - - > 2< 477601792 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68187 - -> 00000080:00200000:1.0:1788201706.616133:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001e2c63e6 00000080:00200000:1.0:1788201706.616135:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.616137:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.616146:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.616159:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.616163:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.616165:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.616175:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.616178:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.616180:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.616184:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616192:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.616200:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.616203:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.616208:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.616211:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116602, 116602] 00000080:00200000:1.0:1788201706.616222:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.616225:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.616256:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.616260:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.616277:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477605888, count: 4096 00000080:00200000:1.0:1788201706.616289:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477605888, bytes: 4096 00000080:00200000:1.0:1788201706.616292:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.616296:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.616297:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477605888, 477609984) 0 0 00000080:00200000:1.0:1788201706.616299:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.616304:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116603, 116603] 00020000:00200000:1.0:1788201706.616307:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477605888 chunk: [477605888, 477609984) 477609984, 4096 00020000:00200000:1.0:1788201706.616309:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.616321:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116603, 116603] 00000008:00010000:1.0:1788201706.616327:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.616329:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.616337:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616346:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477605888 477609983) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616355:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616362:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616370:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.616378:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477605888, 477609984) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.616384:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116603 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.616388:0:17463:0:(rw26.c:898:ll_write_end()) pos 477605888, len 4096, copied 4096 00000080:00200000:1.0:1788201706.616392:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000534a2eaa[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116603] 00000080:00200000:1.0:1788201706.616395:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004b992fc0 fffffc0001029 2:0 ffff93c0c12dc8c0 116603 lru 00000080:00200000:1.0:1788201706.616401:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000819a3459 116603: 1< 2 - - > 2< 477605888 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68188 - -> 00000080:00200000:1.0:1788201706.616403:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000534a2eaa 00000080:00200000:1.0:1788201706.616404:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.616406:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.616415:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.616432:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.616435:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.616437:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.616447:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.616450:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.616453:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.616457:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616465:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.616474:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.616477:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.616481:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.616484:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116603, 116603] 00000080:00200000:1.0:1788201706.616495:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.616499:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.616529:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.616533:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.616549:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477609984, count: 4096 00000080:00200000:1.0:1788201706.616560:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477609984, bytes: 4096 00000080:00200000:1.0:1788201706.616563:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.616567:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.616569:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477609984, 477614080) 0 0 00000080:00200000:1.0:1788201706.616572:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.616577:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116604, 116604] 00020000:00200000:1.0:1788201706.616580:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477609984 chunk: [477609984, 477614080) 477614080, 4096 00020000:00200000:1.0:1788201706.616582:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.616595:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116604, 116604] 00000008:00010000:1.0:1788201706.616601:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.616603:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.616611:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616620:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477609984 477614079) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616629:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616637:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616644:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.616653:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477609984, 477614080) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.616658:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116604 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.616663:0:17463:0:(rw26.c:898:ll_write_end()) pos 477609984, len 4096, copied 4096 00000080:00200000:1.0:1788201706.616667:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001d6269d8[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116604] 00000080:00200000:1.0:1788201706.616670:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bcce884a fffffc0001029 2:0 ffff93c0c12dcee0 116604 lru 00000080:00200000:1.0:1788201706.616675:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000fe6e0fa0 116604: 1< 2 - - > 2< 477609984 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68189 - -> 00000080:00200000:1.0:1788201706.616678:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001d6269d8 00000080:00200000:1.0:1788201706.616679:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.616681:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.616690:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.616702:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.616705:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.616708:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.616717:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.616720:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.616722:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.616726:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616735:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.616744:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.616746:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.616750:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.616754:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116604, 116604] 00000080:00200000:1.0:1788201706.616765:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.616768:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.616799:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.616803:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.616819:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477614080, count: 4096 00000080:00200000:1.0:1788201706.616850:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477614080, bytes: 4096 00000080:00200000:1.0:1788201706.616854:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.616858:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.616859:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477614080, 477618176) 0 0 00000080:00200000:1.0:1788201706.616862:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.616867:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116605, 116605] 00020000:00200000:1.0:1788201706.616870:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477614080 chunk: [477614080, 477618176) 477618176, 4096 00020000:00200000:1.0:1788201706.616872:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.616885:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116605, 116605] 00000008:00010000:1.0:1788201706.616890:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.616893:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.616900:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616910:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477614080 477618175) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616918:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616926:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.616933:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.616941:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477614080, 477618176) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.616946:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116605 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.616951:0:17463:0:(rw26.c:898:ll_write_end()) pos 477614080, len 4096, copied 4096 00000080:00200000:1.0:1788201706.616955:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a5d6c26d[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116605] 00000080:00200000:1.0:1788201706.616958:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000069a00f1 fffffc0001029 2:0 ffff93c0c12dc7e0 116605 lru 00000080:00200000:1.0:1788201706.616964:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000064467049 116605: 1< 2 - - > 2< 477614080 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68190 - -> 00000080:00200000:1.0:1788201706.616967:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a5d6c26d 00000080:00200000:1.0:1788201706.616968:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.616970:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.616979:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.616990:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.616993:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.616995:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.617004:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.617006:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.617009:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.617013:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617033:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.617041:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.617044:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.617048:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.617051:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116605, 116605] 00000080:00200000:1.0:1788201706.617063:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.617066:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.617094:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.617098:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.617114:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477618176, count: 4096 00000080:00200000:1.0:1788201706.617125:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477618176, bytes: 4096 00000080:00200000:1.0:1788201706.617129:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.617132:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.617134:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477618176, 477622272) 0 0 00000080:00200000:1.0:1788201706.617136:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.617141:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116606, 116606] 00020000:00200000:1.0:1788201706.617144:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477618176 chunk: [477618176, 477622272) 477622272, 4096 00020000:00200000:1.0:1788201706.617146:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.617157:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116606, 116606] 00000008:00010000:1.0:1788201706.617162:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.617164:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.617172:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617181:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477618176 477622271) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617189:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617197:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617204:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.617212:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477618176, 477622272) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.617218:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116606 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.617223:0:17463:0:(rw26.c:898:ll_write_end()) pos 477618176, len 4096, copied 4096 00000080:00200000:1.0:1788201706.617227:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002fb1499e[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116606] 00000080:00200000:1.0:1788201706.617230:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000df35391f fffffc0001029 2:0 ffff93c0c12dc2a0 116606 lru 00000080:00200000:1.0:1788201706.617237:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000fb14fefd 116606: 1< 2 - - > 2< 477618176 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68191 - -> 00000080:00200000:1.0:1788201706.617239:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002fb1499e 00000080:00200000:1.0:1788201706.617240:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.617242:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.617251:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.617263:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.617265:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.617267:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.617277:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.617280:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.617283:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.617287:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617295:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.617303:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.617305:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.617309:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.617312:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116606, 116606] 00000080:00200000:1.0:1788201706.617323:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.617326:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.617356:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.617360:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.617375:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477622272, count: 4096 00000080:00200000:1.0:1788201706.617386:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477622272, bytes: 4096 00000080:00200000:1.0:1788201706.617389:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.617393:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.617394:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477622272, 477626368) 0 0 00000080:00200000:1.0:1788201706.617396:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.617402:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116607, 116607] 00020000:00200000:1.0:1788201706.617405:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477622272 chunk: [477622272, 477626368) 477626368, 4096 00020000:00200000:1.0:1788201706.617407:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.617419:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116607, 116607] 00000008:00010000:1.0:1788201706.617425:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.617427:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.617435:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617444:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477622272 477626367) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617452:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617460:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617468:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.617475:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477622272, 477626368) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.617481:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116607 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.617487:0:17463:0:(rw26.c:898:ll_write_end()) pos 477622272, len 4096, copied 4096 00000080:00200000:1.0:1788201706.617491:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a889987a[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116607] 00000080:00200000:1.0:1788201706.617494:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008c7d8ff5 fffffc0001029 2:0 ffff93c0c12dce00 116607 lru 00000080:00200000:1.0:1788201706.617499:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cceec440 116607: 1< 2 - - > 2< 477622272 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68192 - -> 00000080:00200000:1.0:1788201706.617501:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a889987a 00000080:00200000:1.0:1788201706.617503:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.617504:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.617513:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.617525:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.617528:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.617530:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.617539:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.617542:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.617544:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.617548:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617556:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.617564:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.617566:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.617570:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.617573:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116607, 116607] 00000080:00200000:1.0:1788201706.617584:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.617587:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.617616:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.617620:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.617636:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477626368, count: 4096 00000080:00200000:1.0:1788201706.617647:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477626368, bytes: 4096 00000080:00200000:1.0:1788201706.617649:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.617654:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.617655:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477626368, 477630464) 0 0 00000080:00200000:1.0:1788201706.617658:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.617663:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116608, 116608] 00020000:00200000:1.0:1788201706.617666:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477626368 chunk: [477626368, 477630464) 477630464, 4096 00020000:00200000:1.0:1788201706.617668:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.617680:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116608, 116608] 00000008:00010000:1.0:1788201706.617685:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.617687:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.617695:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617704:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477626368 477630463) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617712:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617719:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617727:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.617734:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477626368, 477630464) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.617739:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116608 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.617744:0:17463:0:(rw26.c:898:ll_write_end()) pos 477626368, len 4096, copied 4096 00000080:00200000:1.0:1788201706.617748:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ccfd6c9d[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116608] 00000080:00200000:1.0:1788201706.617751:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d8d33ee1 fffffc0001029 2:0 ffff93c0c12dc1c0 116608 lru 00000080:00200000:1.0:1788201706.617757:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000477bb5dd 116608: 1< 2 - - > 2< 477626368 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68193 - -> 00000080:00200000:1.0:1788201706.617759:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ccfd6c9d 00000080:00200000:1.0:1788201706.617760:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.617762:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.617771:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.617783:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.617786:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.617788:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.617797:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.617800:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.617802:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.617806:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617814:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.617822:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.617824:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.617854:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.617859:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116608, 116608] 00000080:00200000:1.0:1788201706.617872:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.617876:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.617907:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.617911:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.617930:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477630464, count: 4096 00000080:00200000:1.0:1788201706.617941:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477630464, bytes: 4096 00000080:00200000:1.0:1788201706.617944:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.617948:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.617950:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477630464, 477634560) 0 0 00000080:00200000:1.0:1788201706.617953:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.617958:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116609, 116609] 00020000:00200000:1.0:1788201706.617961:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477630464 chunk: [477630464, 477634560) 477634560, 4096 00020000:00200000:1.0:1788201706.617964:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.617976:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116609, 116609] 00000008:00010000:1.0:1788201706.617981:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.617983:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.617990:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.617999:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477630464 477634559) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618008:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618029:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618037:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.618051:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477630464, 477634560) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.618057:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116609 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.618062:0:17463:0:(rw26.c:898:ll_write_end()) pos 477630464, len 4096, copied 4096 00000080:00200000:1.0:1788201706.618066:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000094777f39[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116609] 00000080:00200000:1.0:1788201706.618069:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a1bb4b4d fffffc0001029 2:0 ffff93c0c12dc620 116609 lru 00000080:00200000:1.0:1788201706.618075:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ed3a4f46 116609: 1< 2 - - > 2< 477630464 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68194 - -> 00000080:00200000:1.0:1788201706.618077:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000094777f39 00000080:00200000:1.0:1788201706.618078:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.618080:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.618089:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.618101:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.618104:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.618106:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.618116:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.618119:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.618121:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.618126:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618133:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.618142:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.618144:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.618149:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.618152:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116609, 116609] 00000080:00200000:1.0:1788201706.618163:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.618166:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.618194:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.618197:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.618214:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477634560, count: 4096 00000080:00200000:1.0:1788201706.618225:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477634560, bytes: 4096 00000080:00200000:1.0:1788201706.618228:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.618231:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.618233:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477634560, 477638656) 0 0 00000080:00200000:1.0:1788201706.618235:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.618239:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116610, 116610] 00020000:00200000:1.0:1788201706.618243:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477634560 chunk: [477634560, 477638656) 477638656, 4096 00020000:00200000:1.0:1788201706.618245:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.618256:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116610, 116610] 00000008:00010000:1.0:1788201706.618262:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.618264:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.618271:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618280:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477634560 477638655) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618288:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618296:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618308:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.618316:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477634560, 477638656) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.618321:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116610 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.618327:0:17463:0:(rw26.c:898:ll_write_end()) pos 477634560, len 4096, copied 4096 00000080:00200000:1.0:1788201706.618331:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d89e413a[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116610] 00000080:00200000:1.0:1788201706.618334:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002fe345e7 fffffc0001029 2:0 ffff93c0c12dcd20 116610 lru 00000080:00200000:1.0:1788201706.618340:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000068fdd6ee 116610: 1< 2 - - > 2< 477634560 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68195 - -> 00000080:00200000:1.0:1788201706.618342:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d89e413a 00000080:00200000:1.0:1788201706.618343:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.618345:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.618354:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.618365:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.618367:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.618369:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.618378:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.618381:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.618383:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.618388:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618395:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.618408:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.618410:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.618414:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.618417:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116610, 116610] 00000080:00200000:1.0:1788201706.618427:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.618430:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.618457:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.618461:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.618477:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477638656, count: 4096 00000080:00200000:1.0:1788201706.618488:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477638656, bytes: 4096 00000080:00200000:1.0:1788201706.618491:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.618495:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.618497:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477638656, 477642752) 0 0 00000080:00200000:1.0:1788201706.618499:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.618504:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116611, 116611] 00020000:00200000:1.0:1788201706.618507:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477638656 chunk: [477638656, 477642752) 477642752, 4096 00020000:00200000:1.0:1788201706.618510:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.618522:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116611, 116611] 00000008:00010000:1.0:1788201706.618528:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.618530:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.618538:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618547:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477638656 477642751) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618556:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618569:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618577:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.618585:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477638656, 477642752) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.618590:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116611 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.618595:0:17463:0:(rw26.c:898:ll_write_end()) pos 477638656, len 4096, copied 4096 00000080:00200000:1.0:1788201706.618599:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ddf6b05e[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116611] 00000080:00200000:1.0:1788201706.618602:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bd7a7b9d fffffc0001029 2:0 ffff93c0c12dc9a0 116611 lru 00000080:00200000:1.0:1788201706.618608:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001a0ce8e8 116611: 1< 2 - - > 2< 477638656 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68196 - -> 00000080:00200000:1.0:1788201706.618610:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ddf6b05e 00000080:00200000:1.0:1788201706.618611:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.618613:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.618622:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.618634:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.618637:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.618640:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.618649:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.618652:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.618654:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.618658:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618671:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.618679:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.618682:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.618686:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.618689:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116611, 116611] 00000080:00200000:1.0:1788201706.618701:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.618704:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.618732:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.618735:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.618751:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477642752, count: 4096 00000080:00200000:1.0:1788201706.618762:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477642752, bytes: 4096 00000080:00200000:1.0:1788201706.618765:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.618769:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.618771:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477642752, 477646848) 0 0 00000080:00200000:1.0:1788201706.618773:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.618778:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116612, 116612] 00020000:00200000:1.0:1788201706.618781:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477642752 chunk: [477642752, 477646848) 477646848, 4096 00020000:00200000:1.0:1788201706.618783:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.618796:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116612, 116612] 00000008:00010000:1.0:1788201706.618802:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.618804:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.618811:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618821:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477642752 477646847) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618854:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618862:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618870:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.618878:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477642752, 477646848) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.618883:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116612 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.618889:0:17463:0:(rw26.c:898:ll_write_end()) pos 477642752, len 4096, copied 4096 00000080:00200000:1.0:1788201706.618893:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bca77591[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116612] 00000080:00200000:1.0:1788201706.618896:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000074909af5 fffffc0001029 2:0 ffff93c0c12dcb60 116612 lru 00000080:00200000:1.0:1788201706.618902:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007287cb48 116612: 1< 2 - - > 2< 477642752 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68197 - -> 00000080:00200000:1.0:1788201706.618905:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bca77591 00000080:00200000:1.0:1788201706.618906:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.618908:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.618916:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.618927:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.618930:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.618933:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.618943:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.618946:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.618949:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.618953:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.618961:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.618969:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.618972:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.618976:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.618979:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116612, 116612] 00000080:00200000:1.0:1788201706.618990:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.618993:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.619037:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.619041:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.619059:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477646848, count: 4096 00000080:00200000:1.0:1788201706.619070:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477646848, bytes: 4096 00000080:00200000:1.0:1788201706.619073:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.619077:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.619078:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477646848, 477650944) 0 0 00000080:00200000:1.0:1788201706.619082:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.619087:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116613, 116613] 00020000:00200000:1.0:1788201706.619090:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477646848 chunk: [477646848, 477650944) 477650944, 4096 00020000:00200000:1.0:1788201706.619093:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.619104:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116613, 116613] 00000008:00010000:1.0:1788201706.619109:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.619112:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.619119:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619133:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477646848 477650943) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619142:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619150:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619158:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.619165:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477646848, 477650944) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.619170:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116613 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.619175:0:17463:0:(rw26.c:898:ll_write_end()) pos 477646848, len 4096, copied 4096 00000080:00200000:1.0:1788201706.619179:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b8bd64c3[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116613] 00000080:00200000:1.0:1788201706.619182:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000069996b08 fffffc0001029 2:0 ffff93c0c12dcc40 116613 lru 00000080:00200000:1.0:1788201706.619188:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009fa93331 116613: 1< 2 - - > 2< 477646848 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68198 - -> 00000080:00200000:1.0:1788201706.619190:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b8bd64c3 00000080:00200000:1.0:1788201706.619191:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.619193:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.619202:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.619214:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.619217:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.619219:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.619229:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.619232:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.619234:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.619239:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619247:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.619255:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.619258:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.619262:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.619265:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116613, 116613] 00000080:00200000:1.0:1788201706.619275:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.619278:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.619306:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.619310:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.619326:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477650944, count: 4096 00000080:00200000:1.0:1788201706.619336:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477650944, bytes: 4096 00000080:00200000:1.0:1788201706.619338:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.619342:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.619343:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477650944, 477655040) 0 0 00000080:00200000:1.0:1788201706.619346:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.619351:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116614, 116614] 00020000:00200000:1.0:1788201706.619353:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477650944 chunk: [477650944, 477655040) 477655040, 4096 00020000:00200000:1.0:1788201706.619356:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.619366:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116614, 116614] 00000008:00010000:1.0:1788201706.619371:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.619373:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.619380:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619389:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477650944 477655039) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619398:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619406:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619413:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.619420:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477650944, 477655040) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.619425:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116614 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.619430:0:17463:0:(rw26.c:898:ll_write_end()) pos 477650944, len 4096, copied 4096 00000080:00200000:1.0:1788201706.619434:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004f98fea8[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116614] 00000080:00200000:1.0:1788201706.619437:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006715b7a2 fffffc0001029 2:0 ffff93c0c12dc540 116614 lru 00000080:00200000:1.0:1788201706.619443:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001e7facf1 116614: 1< 2 - - > 2< 477650944 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68199 - -> 00000080:00200000:1.0:1788201706.619445:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004f98fea8 00000080:00200000:1.0:1788201706.619446:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.619448:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.619456:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.619468:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.619471:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.619473:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.619482:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.619484:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.619487:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.619490:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619498:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.619506:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.619509:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.619513:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.619515:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116614, 116614] 00000080:00200000:1.0:1788201706.619525:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.619528:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.619552:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.619555:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.619570:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477655040, count: 4096 00000080:00200000:1.0:1788201706.619579:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477655040, bytes: 4096 00000080:00200000:1.0:1788201706.619582:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.619585:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.619586:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477655040, 477659136) 0 0 00000080:00200000:1.0:1788201706.619588:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.619592:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116615, 116615] 00020000:00200000:1.0:1788201706.619595:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477655040 chunk: [477655040, 477659136) 477659136, 4096 00020000:00200000:1.0:1788201706.619597:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.619606:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116615, 116615] 00000008:00010000:1.0:1788201706.619610:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.619613:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.619619:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619627:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477655040 477659135) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619635:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619642:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619648:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.619655:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477655040, 477659136) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.619659:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116615 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.619663:0:17463:0:(rw26.c:898:ll_write_end()) pos 477655040, len 4096, copied 4096 00000080:00200000:1.0:1788201706.619666:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000457b1e03[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116615] 00000080:00200000:1.0:1788201706.619669:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000299c5fd5 fffffc0001029 2:0 ffff93c0c12dc460 116615 lru 00000080:00200000:1.0:1788201706.619674:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000000c9dd56 116615: 1< 2 - - > 2< 477655040 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68200 - -> 00000080:00200000:1.0:1788201706.619676:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000457b1e03 00000080:00200000:1.0:1788201706.619677:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.619679:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.619686:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.619702:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.619704:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.619706:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.619715:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.619717:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.619720:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.619724:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619732:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.619740:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.619742:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.619746:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.619749:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116615, 116615] 00000080:00200000:1.0:1788201706.619760:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.619763:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.619790:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.619793:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.619809:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477659136, count: 4096 00000080:00200000:1.0:1788201706.619819:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477659136, bytes: 4096 00000080:00200000:1.0:1788201706.619822:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.619825:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.619827:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477659136, 477663232) 0 0 00000080:00200000:1.0:1788201706.619845:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.619850:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116616, 116616] 00020000:00200000:1.0:1788201706.619854:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477659136 chunk: [477659136, 477663232) 477663232, 4096 00020000:00200000:1.0:1788201706.619856:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.619869:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116616, 116616] 00000008:00010000:1.0:1788201706.619874:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.619876:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.619883:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619892:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477659136 477663231) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619901:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619909:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619915:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.619920:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477659136, 477663232) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.619924:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116616 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.619928:0:17463:0:(rw26.c:898:ll_write_end()) pos 477659136, len 4096, copied 4096 00000080:00200000:1.0:1788201706.619931:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005a3e67a8[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116616] 00000080:00200000:1.0:1788201706.619933:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000492e2bb6 fffffc0001029 2:0 ffff93c0c121c1c0 116616 lru 00000080:00200000:1.0:1788201706.619937:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ecb0ef4b 116616: 1< 2 - - > 2< 477659136 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68201 - -> 00000080:00200000:1.0:1788201706.619938:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005a3e67a8 00000080:00200000:1.0:1788201706.619939:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.619941:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.619947:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.619955:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.619957:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.619958:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.619965:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.619967:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.619968:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.619971:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.619976:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.619981:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.619982:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.619985:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.619987:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116616, 116616] 00000080:00200000:1.0:1788201706.619994:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.619995:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.620014:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.620029:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.620041:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477663232, count: 4096 00000080:00200000:1.0:1788201706.620048:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477663232, bytes: 4096 00000080:00200000:1.0:1788201706.620050:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.620053:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.620053:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477663232, 477667328) 0 0 00000080:00200000:1.0:1788201706.620055:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.620059:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116617, 116617] 00020000:00200000:1.0:1788201706.620061:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477663232 chunk: [477663232, 477667328) 477667328, 4096 00020000:00200000:1.0:1788201706.620062:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.620069:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116617, 116617] 00000008:00010000:1.0:1788201706.620072:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.620074:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.620079:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620085:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477663232 477667327) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620091:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620095:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620099:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.620104:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477663232, 477667328) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.620107:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116617 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.620111:0:17463:0:(rw26.c:898:ll_write_end()) pos 477663232, len 4096, copied 4096 00000080:00200000:1.0:1788201706.620113:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000787b48ea[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116617] 00000080:00200000:1.0:1788201706.620115:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d8f4ead7 fffffc0001029 2:0 ffff93c0c121c620 116617 lru 00000080:00200000:1.0:1788201706.620119:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000081b09109 116617: 1< 2 - - > 2< 477663232 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68202 - -> 00000080:00200000:1.0:1788201706.620121:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000787b48ea 00000080:00200000:1.0:1788201706.620122:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.620123:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.620129:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.620137:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.620138:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.620140:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.620146:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.620147:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.620149:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.620151:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620155:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.620160:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.620161:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.620164:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.620166:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116617, 116617] 00000080:00200000:1.0:1788201706.620173:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.620175:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.620192:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.620194:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.620208:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477667328, count: 4096 00000080:00200000:1.0:1788201706.620219:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477667328, bytes: 4096 00000080:00200000:1.0:1788201706.620222:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.620225:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.620227:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477667328, 477671424) 0 0 00000080:00200000:1.0:1788201706.620229:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.620235:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116618, 116618] 00020000:00200000:1.0:1788201706.620237:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477667328 chunk: [477667328, 477671424) 477671424, 4096 00020000:00200000:1.0:1788201706.620240:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.620251:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116618, 116618] 00000008:00010000:1.0:1788201706.620256:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.620258:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.620265:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620273:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477667328 477671423) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620280:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620285:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620289:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.620294:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477667328, 477671424) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.620297:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116618 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.620301:0:17463:0:(rw26.c:898:ll_write_end()) pos 477667328, len 4096, copied 4096 00000080:00200000:1.0:1788201706.620303:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b681985c[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116618] 00000080:00200000:1.0:1788201706.620305:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007f9354ef fffffc0001029 2:0 ffff93c0c121cd20 116618 lru 00000080:00200000:1.0:1788201706.620309:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000da3d05b1 116618: 1< 2 - - > 2< 477667328 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68203 - -> 00000080:00200000:1.0:1788201706.620311:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b681985c 00000080:00200000:1.0:1788201706.620311:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.620313:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.620318:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.620326:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.620328:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.620329:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.620335:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.620337:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.620339:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.620341:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620346:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.620351:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.620353:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.620355:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.620357:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116618, 116618] 00000080:00200000:1.0:1788201706.620364:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.620366:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.620384:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.620386:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.620405:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477671424, count: 4096 00000080:00200000:1.0:1788201706.620414:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477671424, bytes: 4096 00000080:00200000:1.0:1788201706.620416:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.620418:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.620419:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477671424, 477675520) 0 0 00000080:00200000:1.0:1788201706.620421:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.620424:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116619, 116619] 00020000:00200000:1.0:1788201706.620426:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477671424 chunk: [477671424, 477675520) 477675520, 4096 00020000:00200000:1.0:1788201706.620428:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.620434:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116619, 116619] 00000008:00010000:1.0:1788201706.620438:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.620439:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.620445:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620450:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477671424 477675519) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620456:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620461:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620464:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.620469:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477671424, 477675520) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.620472:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116619 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.620476:0:17463:0:(rw26.c:898:ll_write_end()) pos 477671424, len 4096, copied 4096 00000080:00200000:1.0:1788201706.620478:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000055538f2b[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116619] 00000080:00200000:1.0:1788201706.620480:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bb596aeb fffffc0001029 2:0 ffff93c0c121c9a0 116619 lru 00000080:00200000:1.0:1788201706.620484:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000024aed23a 116619: 1< 2 - - > 2< 477671424 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68204 - -> 00000080:00200000:1.0:1788201706.620485:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000055538f2b 00000080:00200000:1.0:1788201706.620486:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.620487:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.620493:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.620501:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.620503:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.620504:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.620510:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.620511:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.620513:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.620515:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620520:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.620525:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.620527:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.620529:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.620531:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116619, 116619] 00000080:00200000:1.0:1788201706.620537:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.620539:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.620557:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.620559:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.620569:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477675520, count: 4096 00000080:00200000:1.0:1788201706.620575:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477675520, bytes: 4096 00000080:00200000:1.0:1788201706.620577:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.620579:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.620580:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477675520, 477679616) 0 0 00000080:00200000:1.0:1788201706.620581:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.620584:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116620, 116620] 00020000:00200000:1.0:1788201706.620586:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477675520 chunk: [477675520, 477679616) 477679616, 4096 00020000:00200000:1.0:1788201706.620587:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.620594:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116620, 116620] 00000008:00010000:1.0:1788201706.620597:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.620598:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.620603:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620608:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477675520 477679615) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620614:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620618:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620622:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.620626:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477675520, 477679616) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.620629:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116620 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.620633:0:17463:0:(rw26.c:898:ll_write_end()) pos 477675520, len 4096, copied 4096 00000080:00200000:1.0:1788201706.620635:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002e15b5f3[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116620] 00000080:00200000:1.0:1788201706.620637:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000dfe81a9 fffffc0001029 2:0 ffff93c0c121cb60 116620 lru 00000080:00200000:1.0:1788201706.620641:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e8a1ae6c 116620: 1< 2 - - > 2< 477675520 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68205 - -> 00000080:00200000:1.0:1788201706.620642:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002e15b5f3 00000080:00200000:1.0:1788201706.620643:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.620644:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.620650:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.620657:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.620659:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.620660:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.620666:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.620668:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.620669:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.620671:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620676:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.620681:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.620682:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.620685:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.620687:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116620, 116620] 00000080:00200000:1.0:1788201706.620693:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.620695:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.620712:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.620714:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.620724:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477679616, count: 4096 00000080:00200000:1.0:1788201706.620730:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477679616, bytes: 4096 00000080:00200000:1.0:1788201706.620732:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.620734:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.620735:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477679616, 477683712) 0 0 00000080:00200000:1.0:1788201706.620737:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.620739:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116621, 116621] 00020000:00200000:1.0:1788201706.620741:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477679616 chunk: [477679616, 477683712) 477683712, 4096 00020000:00200000:1.0:1788201706.620743:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.620749:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116621, 116621] 00000008:00010000:1.0:1788201706.620752:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.620753:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.620758:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620764:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477679616 477683711) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620769:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620773:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620777:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.620785:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477679616, 477683712) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.620788:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116621 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.620791:0:17463:0:(rw26.c:898:ll_write_end()) pos 477679616, len 4096, copied 4096 00000080:00200000:1.0:1788201706.620793:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004e9d506c[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116621] 00000080:00200000:1.0:1788201706.620795:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d1f413ad fffffc0001029 2:0 ffff93c0c121cc40 116621 lru 00000080:00200000:1.0:1788201706.620799:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002edb5938 116621: 1< 2 - - > 2< 477679616 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68206 - -> 00000080:00200000:1.0:1788201706.620800:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004e9d506c 00000080:00200000:1.0:1788201706.620801:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.620802:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.620808:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.620815:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.620817:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.620818:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.620824:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.620825:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.620827:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.620846:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620851:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.620857:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.620858:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.620862:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.620864:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116621, 116621] 00000080:00200000:1.0:1788201706.620873:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.620875:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.620894:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.620896:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.620906:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477683712, count: 4096 00000080:00200000:1.0:1788201706.620912:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477683712, bytes: 4096 00000080:00200000:1.0:1788201706.620913:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.620916:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.620917:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477683712, 477687808) 0 0 00000080:00200000:1.0:1788201706.620918:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.620921:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116622, 116622] 00020000:00200000:1.0:1788201706.620923:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477683712 chunk: [477683712, 477687808) 477687808, 4096 00020000:00200000:1.0:1788201706.620925:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.620931:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116622, 116622] 00000008:00010000:1.0:1788201706.620934:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.620935:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.620940:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620946:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477683712 477687807) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620951:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620956:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.620963:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.620968:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477683712, 477687808) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.620970:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116622 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.620974:0:17463:0:(rw26.c:898:ll_write_end()) pos 477683712, len 4096, copied 4096 00000080:00200000:1.0:1788201706.620976:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000014b9bc01[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116622] 00000080:00200000:1.0:1788201706.620978:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000150b5778 fffffc0001029 2:0 ffff93c0c121c540 116622 lru 00000080:00200000:1.0:1788201706.620981:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000058f8a7c 116622: 1< 2 - - > 2< 477683712 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68207 - -> 00000080:00200000:1.0:1788201706.620982:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000014b9bc01 00000080:00200000:1.0:1788201706.620983:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.620985:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.620990:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.620998:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.620999:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.621001:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.621007:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.621008:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.621010:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.621012:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621030:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.621039:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.621041:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.621044:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.621046:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116622, 116622] 00000080:00200000:1.0:1788201706.621053:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.621055:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.621073:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.621075:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.621085:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477687808, count: 4096 00000080:00200000:1.0:1788201706.621091:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477687808, bytes: 4096 00000080:00200000:1.0:1788201706.621093:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.621095:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.621096:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477687808, 477691904) 0 0 00000080:00200000:1.0:1788201706.621097:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.621101:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116623, 116623] 00020000:00200000:1.0:1788201706.621103:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477687808 chunk: [477687808, 477691904) 477691904, 4096 00020000:00200000:1.0:1788201706.621105:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.621111:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116623, 116623] 00000008:00010000:1.0:1788201706.621114:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.621116:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.621121:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621126:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477687808 477691903) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621132:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621141:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621145:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.621149:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477687808, 477691904) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.621152:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116623 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.621155:0:17463:0:(rw26.c:898:ll_write_end()) pos 477687808, len 4096, copied 4096 00000080:00200000:1.0:1788201706.621157:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000faa5d04d[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116623] 00000080:00200000:1.0:1788201706.621159:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007041d493 fffffc0001029 2:0 ffff93c0c121c460 116623 lru 00000080:00200000:1.0:1788201706.621162:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008bb35efd 116623: 1< 2 - - > 2< 477687808 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68208 - -> 00000080:00200000:1.0:1788201706.621163:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000faa5d04d 00000080:00200000:1.0:1788201706.621164:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.621165:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.621171:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.621179:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.621180:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.621181:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.621187:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.621189:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.621190:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.621193:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621203:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.621211:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.621214:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.621218:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.621221:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116623, 116623] 00000080:00200000:1.0:1788201706.621232:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.621236:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.621262:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.621266:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.621281:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477691904, count: 4096 00000080:00200000:1.0:1788201706.621291:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477691904, bytes: 4096 00000080:00200000:1.0:1788201706.621293:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.621296:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.621297:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477691904, 477696000) 0 0 00000080:00200000:1.0:1788201706.621298:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.621302:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116624, 116624] 00020000:00200000:1.0:1788201706.621304:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477691904 chunk: [477691904, 477696000) 477696000, 4096 00020000:00200000:1.0:1788201706.621305:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.621312:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116624, 116624] 00000008:00010000:1.0:1788201706.621316:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.621317:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.621322:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621328:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477691904 477695999) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621337:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621342:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621346:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.621350:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477691904, 477696000) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.621353:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116624 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.621357:0:17463:0:(rw26.c:898:ll_write_end()) pos 477691904, len 4096, copied 4096 00000080:00200000:1.0:1788201706.621359:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fc3896f5[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116624] 00000080:00200000:1.0:1788201706.621361:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ea67bed4 fffffc0001029 2:0 ffff93c0c121c0e0 116624 lru 00000080:00200000:1.0:1788201706.621364:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000195ffce4 116624: 1< 2 - - > 2< 477691904 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68209 - -> 00000080:00200000:1.0:1788201706.621366:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fc3896f5 00000080:00200000:1.0:1788201706.621366:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.621368:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.621374:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.621381:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.621383:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.621384:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.621390:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.621392:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.621394:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.621396:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621401:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.621406:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.621407:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.621410:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.621412:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116624, 116624] 00000080:00200000:1.0:1788201706.621419:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.621420:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.621439:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.621441:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.621451:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477696000, count: 4096 00000080:00200000:1.0:1788201706.621457:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477696000, bytes: 4096 00000080:00200000:1.0:1788201706.621459:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.621461:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.621462:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477696000, 477700096) 0 0 00000080:00200000:1.0:1788201706.621463:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.621467:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116625, 116625] 00020000:00200000:1.0:1788201706.621468:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477696000 chunk: [477696000, 477700096) 477700096, 4096 00020000:00200000:1.0:1788201706.621470:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.621476:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116625, 116625] 00000008:00010000:1.0:1788201706.621479:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.621480:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.621485:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621494:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477696000 477700095) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621499:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621504:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621507:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.621511:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477696000, 477700096) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.621514:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116625 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.621518:0:17463:0:(rw26.c:898:ll_write_end()) pos 477696000, len 4096, copied 4096 00000080:00200000:1.0:1788201706.621520:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005e0afc24[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116625] 00000080:00200000:1.0:1788201706.621522:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000045df69ef fffffc0001029 2:0 ffff93c0c121c700 116625 lru 00000080:00200000:1.0:1788201706.621525:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000942aca23 116625: 1< 2 - - > 2< 477696000 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68210 - -> 00000080:00200000:1.0:1788201706.621527:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005e0afc24 00000080:00200000:1.0:1788201706.621527:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.621528:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.621534:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.621541:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.621543:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.621544:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.621550:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.621552:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.621554:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.621556:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621561:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.621566:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.621568:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.621570:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.621572:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116625, 116625] 00000080:00200000:1.0:1788201706.621578:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.621580:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.621598:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.621600:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.621610:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477700096, count: 4096 00000080:00200000:1.0:1788201706.621616:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477700096, bytes: 4096 00000080:00200000:1.0:1788201706.621618:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.621620:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.621621:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477700096, 477704192) 0 0 00000080:00200000:1.0:1788201706.621622:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.621625:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116626, 116626] 00020000:00200000:1.0:1788201706.621627:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477700096 chunk: [477700096, 477704192) 477704192, 4096 00020000:00200000:1.0:1788201706.621629:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.621635:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116626, 116626] 00000008:00010000:1.0:1788201706.621638:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.621639:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.621644:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621650:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477700096 477704191) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621656:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621660:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621664:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.621669:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477700096, 477704192) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.621672:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116626 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.621675:0:17463:0:(rw26.c:898:ll_write_end()) pos 477700096, len 4096, copied 4096 00000080:00200000:1.0:1788201706.621678:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000097bbacd1[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116626] 00000080:00200000:1.0:1788201706.621680:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ad2b96de fffffc0001029 2:0 ffff93c0c121c000 116626 lru 00000080:00200000:1.0:1788201706.621683:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000df0fefed 116626: 1< 2 - - > 2< 477700096 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68211 - -> 00000080:00200000:1.0:1788201706.621685:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000097bbacd1 00000080:00200000:1.0:1788201706.621685:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.621687:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.621692:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.621699:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.621701:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.621703:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.621708:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.621710:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.621711:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.621714:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621718:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.621723:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.621724:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.621727:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.621729:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116626, 116626] 00000080:00200000:1.0:1788201706.621735:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.621737:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.621755:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.621757:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.621767:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477704192, count: 4096 00000080:00200000:1.0:1788201706.621773:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477704192, bytes: 4096 00000080:00200000:1.0:1788201706.621774:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.621776:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.621778:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477704192, 477708288) 0 0 00000080:00200000:1.0:1788201706.621779:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.621782:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116627, 116627] 00020000:00200000:1.0:1788201706.621784:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477704192 chunk: [477704192, 477708288) 477708288, 4096 00020000:00200000:1.0:1788201706.621786:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.621792:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116627, 116627] 00000008:00010000:1.0:1788201706.621795:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.621797:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.621802:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621807:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477704192 477708287) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621812:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621817:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621821:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.621825:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477704192, 477708288) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.621827:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116627 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.621844:0:17463:0:(rw26.c:898:ll_write_end()) pos 477704192, len 4096, copied 4096 00000080:00200000:1.0:1788201706.621847:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c8cef36a[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116627] 00000080:00200000:1.0:1788201706.621849:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009af844a7 fffffc0001029 2:0 ffff93c0c121ca80 116627 lru 00000080:00200000:1.0:1788201706.621852:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000966d3cc2 116627: 1< 2 - - > 2< 477704192 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68212 - -> 00000080:00200000:1.0:1788201706.621853:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c8cef36a 00000080:00200000:1.0:1788201706.621854:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.621855:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.621861:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.621872:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.621874:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.621875:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.621881:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.621883:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.621884:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.621886:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621891:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.621897:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.621898:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.621900:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.621902:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116627, 116627] 00000080:00200000:1.0:1788201706.621909:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.621911:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.621928:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.621930:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.621940:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477708288, count: 4096 00000080:00200000:1.0:1788201706.621946:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477708288, bytes: 4096 00000080:00200000:1.0:1788201706.621948:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.621950:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.621951:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477708288, 477712384) 0 0 00000080:00200000:1.0:1788201706.621953:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.621956:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116628, 116628] 00020000:00200000:1.0:1788201706.621958:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477708288 chunk: [477708288, 477712384) 477712384, 4096 00020000:00200000:1.0:1788201706.621959:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.621966:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116628, 116628] 00000008:00010000:1.0:1788201706.621968:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.621970:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.621975:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621980:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477708288 477712383) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621985:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621990:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.621994:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.621999:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477708288, 477712384) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.622001:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116628 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.622005:0:17463:0:(rw26.c:898:ll_write_end()) pos 477708288, len 4096, copied 4096 00000080:00200000:1.0:1788201706.622007:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000065d58526[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116628] 00000080:00200000:1.0:1788201706.622009:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000007877905 fffffc0001029 2:0 ffff93c0c121c380 116628 lru 00000080:00200000:1.0:1788201706.622013:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bad93bf9 116628: 1< 2 - - > 2< 477708288 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68213 - -> 00000080:00200000:1.0:1788201706.622014:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000065d58526 00000080:00200000:1.0:1788201706.622027:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.622029:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.622036:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.622044:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.622046:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.622047:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.622054:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.622055:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.622057:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.622059:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622064:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.622069:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.622071:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.622073:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.622075:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116628, 116628] 00000080:00200000:1.0:1788201706.622082:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.622084:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.622101:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.622103:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.622113:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477712384, count: 4096 00000080:00200000:1.0:1788201706.622120:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477712384, bytes: 4096 00000080:00200000:1.0:1788201706.622121:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.622123:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.622124:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477712384, 477716480) 0 0 00000080:00200000:1.0:1788201706.622126:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.622130:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116629, 116629] 00020000:00200000:1.0:1788201706.622132:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477712384 chunk: [477712384, 477716480) 477716480, 4096 00020000:00200000:1.0:1788201706.622133:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.622140:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116629, 116629] 00000008:00010000:1.0:1788201706.622142:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.622144:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.622149:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622154:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477712384 477716479) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622160:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622164:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622168:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.622172:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477712384, 477716480) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.622175:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116629 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.622178:0:17463:0:(rw26.c:898:ll_write_end()) pos 477712384, len 4096, copied 4096 00000080:00200000:1.0:1788201706.622181:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008476fb71[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116629] 00000080:00200000:1.0:1788201706.622182:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f9341e0c fffffc0001029 2:0 ffff93c0c121c8c0 116629 lru 00000080:00200000:1.0:1788201706.622186:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003357d394 116629: 1< 2 - - > 2< 477712384 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68214 - -> 00000080:00200000:1.0:1788201706.622188:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008476fb71 00000080:00200000:1.0:1788201706.622189:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.622190:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.622196:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.622206:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.622209:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.622211:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.622221:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.622223:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.622226:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.622230:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622238:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.622246:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.622248:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.622251:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.622253:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116629, 116629] 00000080:00200000:1.0:1788201706.622260:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.622263:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.622281:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.622283:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.622294:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477716480, count: 4096 00000080:00200000:1.0:1788201706.622300:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477716480, bytes: 4096 00000080:00200000:1.0:1788201706.622302:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.622304:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.622305:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477716480, 477720576) 0 0 00000080:00200000:1.0:1788201706.622306:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.622310:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116630, 116630] 00020000:00200000:1.0:1788201706.622312:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477716480 chunk: [477716480, 477720576) 477720576, 4096 00020000:00200000:1.0:1788201706.622313:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.622320:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116630, 116630] 00000008:00010000:1.0:1788201706.622323:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.622324:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.622329:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622335:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477716480 477720575) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622341:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622345:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622349:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.622353:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477716480, 477720576) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.622356:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116630 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.622359:0:17463:0:(rw26.c:898:ll_write_end()) pos 477716480, len 4096, copied 4096 00000080:00200000:1.0:1788201706.622362:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007bfaab15[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116630] 00000080:00200000:1.0:1788201706.622364:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a35908fc fffffc0001029 2:0 ffff93c0c121cee0 116630 lru 00000080:00200000:1.0:1788201706.622367:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001e882031 116630: 1< 2 - - > 2< 477716480 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68215 - -> 00000080:00200000:1.0:1788201706.622369:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007bfaab15 00000080:00200000:1.0:1788201706.622369:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.622371:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.622376:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.622384:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.622385:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.622387:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.622392:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.622394:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.622396:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.622398:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622402:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.622407:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.622408:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.622411:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.622413:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116630, 116630] 00000080:00200000:1.0:1788201706.622420:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.622422:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.622439:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.622441:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.622451:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477720576, count: 4096 00000080:00200000:1.0:1788201706.622457:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477720576, bytes: 4096 00000080:00200000:1.0:1788201706.622459:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.622461:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.622462:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477720576, 477724672) 0 0 00000080:00200000:1.0:1788201706.622463:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.622466:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116631, 116631] 00020000:00200000:1.0:1788201706.622468:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477720576 chunk: [477720576, 477724672) 477724672, 4096 00020000:00200000:1.0:1788201706.622469:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.622476:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116631, 116631] 00000008:00010000:1.0:1788201706.622479:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.622480:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.622485:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622490:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477720576 477724671) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622495:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622500:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622504:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.622508:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477720576, 477724672) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.622510:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116631 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.622514:0:17463:0:(rw26.c:898:ll_write_end()) pos 477720576, len 4096, copied 4096 00000080:00200000:1.0:1788201706.622516:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009f61adfb[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116631] 00000080:00200000:1.0:1788201706.622518:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000043f654d9 fffffc0001029 2:0 ffff93c0c121c7e0 116631 lru 00000080:00200000:1.0:1788201706.622522:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002e18cfe4 116631: 1< 2 - - > 2< 477720576 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68216 - -> 00000080:00200000:1.0:1788201706.622523:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009f61adfb 00000080:00200000:1.0:1788201706.622524:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.622525:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.622531:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.622538:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.622540:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.622541:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.622547:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.622548:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.622550:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.622552:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622557:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.622561:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.622563:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.622565:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.622567:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116631, 116631] 00000080:00200000:1.0:1788201706.622574:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.622576:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.622593:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.622595:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.622605:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477724672, count: 4096 00000080:00200000:1.0:1788201706.622611:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477724672, bytes: 4096 00000080:00200000:1.0:1788201706.622613:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.622615:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.622616:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477724672, 477728768) 0 0 00000080:00200000:1.0:1788201706.622617:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.622620:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116632, 116632] 00020000:00200000:1.0:1788201706.622622:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477724672 chunk: [477724672, 477728768) 477728768, 4096 00020000:00200000:1.0:1788201706.622624:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.622630:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116632, 116632] 00000008:00010000:1.0:1788201706.622633:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.622634:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.622639:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622644:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477724672 477728767) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622650:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622654:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622658:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.622662:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477724672, 477728768) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.622666:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116632 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.622669:0:17463:0:(rw26.c:898:ll_write_end()) pos 477724672, len 4096, copied 4096 00000080:00200000:1.0:1788201706.622671:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000674fdf7d[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116632] 00000080:00200000:1.0:1788201706.622673:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b5624b45 fffffc0001029 2:0 ffff93c0c121c2a0 116632 lru 00000080:00200000:1.0:1788201706.622677:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004c8f7221 116632: 1< 2 - - > 2< 477724672 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68217 - -> 00000080:00200000:1.0:1788201706.622678:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000674fdf7d 00000080:00200000:1.0:1788201706.622679:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.622680:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.622686:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.622693:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.622695:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.622696:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.622702:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.622703:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.622705:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.622707:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622712:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.622716:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.622718:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.622720:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.622722:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116632, 116632] 00000080:00200000:1.0:1788201706.622728:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.622730:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.622747:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.622750:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.622760:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477728768, count: 4096 00000080:00200000:1.0:1788201706.622765:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477728768, bytes: 4096 00000080:00200000:1.0:1788201706.622767:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.622769:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.622770:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477728768, 477732864) 0 0 00000080:00200000:1.0:1788201706.622772:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.622775:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116633, 116633] 00020000:00200000:1.0:1788201706.622776:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477728768 chunk: [477728768, 477732864) 477732864, 4096 00020000:00200000:1.0:1788201706.622778:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.622784:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116633, 116633] 00000008:00010000:1.0:1788201706.622787:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.622788:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.622793:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622799:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477728768 477732863) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622804:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622808:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622812:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.622820:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477728768, 477732864) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.622823:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116633 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.622827:0:17463:0:(rw26.c:898:ll_write_end()) pos 477728768, len 4096, copied 4096 00000080:00200000:1.0:1788201706.622841:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f30379ba[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116633] 00000080:00200000:1.0:1788201706.622844:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ef1b382b fffffc0001029 2:0 ffff93c0c121ce00 116633 lru 00000080:00200000:1.0:1788201706.622847:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001ba4eed3 116633: 1< 2 - - > 2< 477728768 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68218 - -> 00000080:00200000:1.0:1788201706.622849:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f30379ba 00000080:00200000:1.0:1788201706.622849:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.622851:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.622856:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.622864:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.622865:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.622867:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.622872:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.622874:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.622876:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.622878:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622883:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.622888:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.622889:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.622892:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.622894:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116633, 116633] 00000080:00200000:1.0:1788201706.622900:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.622902:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.622920:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.622922:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.622932:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477732864, count: 4096 00000080:00200000:1.0:1788201706.622938:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477732864, bytes: 4096 00000080:00200000:1.0:1788201706.622940:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.622942:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.622943:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477732864, 477736960) 0 0 00000080:00200000:1.0:1788201706.622944:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.622947:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116634, 116634] 00020000:00200000:1.0:1788201706.622949:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477732864 chunk: [477732864, 477736960) 477736960, 4096 00020000:00200000:1.0:1788201706.622951:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.622957:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116634, 116634] 00000008:00010000:1.0:1788201706.622960:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.622961:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.622966:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622972:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477732864 477736959) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622977:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622981:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.622989:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.622993:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477732864, 477736960) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.622996:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116634 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.622999:0:17463:0:(rw26.c:898:ll_write_end()) pos 477732864, len 4096, copied 4096 00000080:00200000:1.0:1788201706.623001:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e4a765e2[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116634] 00000080:00200000:1.0:1788201706.623003:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002e4c5857 fffffc0001029 2:0 ffff93c0c121d380 116634 lru 00000080:00200000:1.0:1788201706.623006:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000074d87ced 116634: 1< 2 - - > 2< 477732864 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68219 - -> 00000080:00200000:1.0:1788201706.623007:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e4a765e2 00000080:00200000:1.0:1788201706.623008:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.623009:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.623026:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.623035:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.623037:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.623039:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.623045:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.623047:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.623049:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.623051:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623056:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.623064:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.623066:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.623068:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.623070:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116634, 116634] 00000080:00200000:1.0:1788201706.623077:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.623079:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.623097:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.623099:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.623109:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477736960, count: 4096 00000080:00200000:1.0:1788201706.623115:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477736960, bytes: 4096 00000080:00200000:1.0:1788201706.623117:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.623119:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.623120:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477736960, 477741056) 0 0 00000080:00200000:1.0:1788201706.623122:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.623125:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116635, 116635] 00020000:00200000:1.0:1788201706.623127:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477736960 chunk: [477736960, 477741056) 477741056, 4096 00020000:00200000:1.0:1788201706.623128:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.623134:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116635, 116635] 00000008:00010000:1.0:1788201706.623138:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.623139:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.623144:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623149:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477736960 477741055) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623155:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623163:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623167:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.623171:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477736960, 477741056) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.623174:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116635 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.623178:0:17463:0:(rw26.c:898:ll_write_end()) pos 477736960, len 4096, copied 4096 00000080:00200000:1.0:1788201706.623180:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fe139ac8[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116635] 00000080:00200000:1.0:1788201706.623181:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000beac42b7 fffffc0001029 2:0 ffff93c0c121d8c0 116635 lru 00000080:00200000:1.0:1788201706.623185:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000009d98805 116635: 1< 2 - - > 2< 477736960 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68220 - -> 00000080:00200000:1.0:1788201706.623186:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fe139ac8 00000080:00200000:1.0:1788201706.623186:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.623188:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.623193:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.623203:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.623206:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.623208:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.623217:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.623220:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.623222:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.623226:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623239:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.623246:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.623249:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.623253:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.623256:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116635, 116635] 00000080:00200000:1.0:1788201706.623265:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.623269:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.623294:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.623297:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.623313:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477741056, count: 4096 00000080:00200000:1.0:1788201706.623320:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477741056, bytes: 4096 00000080:00200000:1.0:1788201706.623322:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.623324:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.623325:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477741056, 477745152) 0 0 00000080:00200000:1.0:1788201706.623327:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.623330:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116636, 116636] 00020000:00200000:1.0:1788201706.623332:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477741056 chunk: [477741056, 477745152) 477745152, 4096 00020000:00200000:1.0:1788201706.623334:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.623341:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116636, 116636] 00000008:00010000:1.0:1788201706.623344:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.623345:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.623350:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623356:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477741056 477745151) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623365:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623370:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623374:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.623378:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477741056, 477745152) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.623381:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116636 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.623384:0:17463:0:(rw26.c:898:ll_write_end()) pos 477741056, len 4096, copied 4096 00000080:00200000:1.0:1788201706.623387:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000123f2b61[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116636] 00000080:00200000:1.0:1788201706.623388:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000add1d745 fffffc0001029 2:0 ffff93c0c121dee0 116636 lru 00000080:00200000:1.0:1788201706.623392:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000037b3e350 116636: 1< 2 - - > 2< 477741056 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68221 - -> 00000080:00200000:1.0:1788201706.623393:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000123f2b61 00000080:00200000:1.0:1788201706.623394:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.623395:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.623401:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.623408:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.623410:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.623411:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.623417:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.623419:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.623420:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.623423:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623427:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.623432:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.623434:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.623436:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.623438:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116636, 116636] 00000080:00200000:1.0:1788201706.623445:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.623446:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.623464:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.623466:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.623476:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477745152, count: 4096 00000080:00200000:1.0:1788201706.623482:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477745152, bytes: 4096 00000080:00200000:1.0:1788201706.623484:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.623486:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.623487:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477745152, 477749248) 0 0 00000080:00200000:1.0:1788201706.623489:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.623492:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116637, 116637] 00020000:00200000:1.0:1788201706.623494:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477745152 chunk: [477745152, 477749248) 477749248, 4096 00020000:00200000:1.0:1788201706.623495:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.623501:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116637, 116637] 00000008:00010000:1.0:1788201706.623505:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.623506:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.623511:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623519:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477745152 477749247) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623524:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623528:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623533:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.623537:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477745152, 477749248) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.623540:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116637 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.623543:0:17463:0:(rw26.c:898:ll_write_end()) pos 477745152, len 4096, copied 4096 00000080:00200000:1.0:1788201706.623546:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000071288cc1[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116637] 00000080:00200000:1.0:1788201706.623547:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000091aaf0ac fffffc0001029 2:0 ffff93c0c121d7e0 116637 lru 00000080:00200000:1.0:1788201706.623551:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002307bc89 116637: 1< 2 - - > 2< 477745152 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68222 - -> 00000080:00200000:1.0:1788201706.623552:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000071288cc1 00000080:00200000:1.0:1788201706.623553:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.623554:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.623559:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.623567:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.623569:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.623570:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.623576:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.623578:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.623579:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.623582:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623586:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.623591:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.623593:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.623595:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.623596:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116637, 116637] 00000080:00200000:1.0:1788201706.623603:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.623605:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.623622:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.623625:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.623634:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477749248, count: 4096 00000080:00200000:1.0:1788201706.623640:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477749248, bytes: 4096 00000080:00200000:1.0:1788201706.623642:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.623644:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.623645:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477749248, 477753344) 0 0 00000080:00200000:1.0:1788201706.623647:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.623650:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116638, 116638] 00020000:00200000:1.0:1788201706.623651:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477749248 chunk: [477749248, 477753344) 477753344, 4096 00020000:00200000:1.0:1788201706.623653:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.623659:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116638, 116638] 00000008:00010000:1.0:1788201706.623662:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.623663:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.623668:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623674:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477749248 477753343) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623679:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623683:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623687:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.623692:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477749248, 477753344) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.623695:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116638 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.623698:0:17463:0:(rw26.c:898:ll_write_end()) pos 477749248, len 4096, copied 4096 00000080:00200000:1.0:1788201706.623700:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000015a34613[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116638] 00000080:00200000:1.0:1788201706.623702:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001b06991e fffffc0001029 2:0 ffff93c0c121d2a0 116638 lru 00000080:00200000:1.0:1788201706.623705:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f3264524 116638: 1< 2 - - > 2< 477749248 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68223 - -> 00000080:00200000:1.0:1788201706.623707:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000015a34613 00000080:00200000:1.0:1788201706.623707:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.623709:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.623714:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.623721:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.623723:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.623724:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.623730:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.623732:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.623733:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.623735:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623740:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.623744:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.623746:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.623748:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.623750:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116638, 116638] 00000080:00200000:1.0:1788201706.623757:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.623759:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.623776:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.623778:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.623788:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477753344, count: 4096 00000080:00200000:1.0:1788201706.623794:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477753344, bytes: 4096 00000080:00200000:1.0:1788201706.623796:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.623798:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.623799:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477753344, 477757440) 0 0 00000080:00200000:1.0:1788201706.623800:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.623803:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116639, 116639] 00020000:00200000:1.0:1788201706.623805:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477753344 chunk: [477753344, 477757440) 477757440, 4096 00020000:00200000:1.0:1788201706.623806:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.623813:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116639, 116639] 00000008:00010000:1.0:1788201706.623816:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.623818:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.623822:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623828:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477753344 477757439) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623846:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623851:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623856:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.623860:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477753344, 477757440) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.623863:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116639 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.623867:0:17463:0:(rw26.c:898:ll_write_end()) pos 477753344, len 4096, copied 4096 00000080:00200000:1.0:1788201706.623869:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d52a8d02[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116639] 00000080:00200000:1.0:1788201706.623871:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b12dfc8a fffffc0001029 2:0 ffff93c0c121de00 116639 lru 00000080:00200000:1.0:1788201706.623874:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000611eff8c 116639: 1< 2 - - > 2< 477753344 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68224 - -> 00000080:00200000:1.0:1788201706.623875:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d52a8d02 00000080:00200000:1.0:1788201706.623876:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.623877:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.623883:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.623894:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.623895:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.623897:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.623903:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.623904:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.623906:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.623908:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.623913:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.623917:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.623919:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.623921:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.623923:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116639, 116639] 00000080:00200000:1.0:1788201706.623930:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.623932:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.623949:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.623951:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.623961:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477757440, count: 4096 00000080:00200000:1.0:1788201706.623967:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477757440, bytes: 4096 00000080:00200000:1.0:1788201706.623969:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.623971:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.623972:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477757440, 477761536) 0 0 00000080:00200000:1.0:1788201706.623973:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.623976:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116640, 116640] 00020000:00200000:1.0:1788201706.623979:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477757440 chunk: [477757440, 477761536) 477761536, 4096 00020000:00200000:1.0:1788201706.623980:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.623986:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116640, 116640] 00000008:00010000:1.0:1788201706.623989:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.623991:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.623995:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624001:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477757440 477761535) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624006:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624011:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624027:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.624034:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477757440, 477761536) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.624038:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116640 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.624042:0:17463:0:(rw26.c:898:ll_write_end()) pos 477757440, len 4096, copied 4096 00000080:00200000:1.0:1788201706.624044:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000cc84c791[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116640] 00000080:00200000:1.0:1788201706.624046:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d388c987 fffffc0001029 2:0 ffff93c0c121d1c0 116640 lru 00000080:00200000:1.0:1788201706.624050:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006e215b37 116640: 1< 2 - - > 2< 477757440 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68225 - -> 00000080:00200000:1.0:1788201706.624051:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000cc84c791 00000080:00200000:1.0:1788201706.624052:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.624053:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.624059:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.624067:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.624069:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.624070:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.624076:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.624078:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.624079:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.624082:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624087:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.624092:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.624093:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.624095:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.624097:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116640, 116640] 00000080:00200000:1.0:1788201706.624104:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.624106:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.624124:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.624126:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.624136:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477761536, count: 4096 00000080:00200000:1.0:1788201706.624142:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477761536, bytes: 4096 00000080:00200000:1.0:1788201706.624143:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.624146:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.624147:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477761536, 477765632) 0 0 00000080:00200000:1.0:1788201706.624148:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.624151:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116641, 116641] 00020000:00200000:1.0:1788201706.624153:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477761536 chunk: [477761536, 477765632) 477765632, 4096 00020000:00200000:1.0:1788201706.624155:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.624161:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116641, 116641] 00000008:00010000:1.0:1788201706.624164:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.624166:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.624171:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624176:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477761536 477765631) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624182:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624186:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624190:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.624194:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477761536, 477765632) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.624197:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116641 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.624202:0:17463:0:(rw26.c:898:ll_write_end()) pos 477761536, len 4096, copied 4096 00000080:00200000:1.0:1788201706.624205:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007d86c75b[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116641] 00000080:00200000:1.0:1788201706.624208:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000008950dc0 fffffc0001029 2:0 ffff93c0c121d620 116641 lru 00000080:00200000:1.0:1788201706.624214:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000093b29a42 116641: 1< 2 - - > 2< 477761536 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68226 - -> 00000080:00200000:1.0:1788201706.624217:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007d86c75b 00000080:00200000:1.0:1788201706.624219:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.624221:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.624229:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.624241:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.624244:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.624247:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.624255:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.624257:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.624259:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.624261:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624266:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.624271:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.624273:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.624275:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.624277:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116641, 116641] 00000080:00200000:1.0:1788201706.624284:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.624286:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.624304:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.624306:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.624316:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477765632, count: 4096 00000080:00200000:1.0:1788201706.624322:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477765632, bytes: 4096 00000080:00200000:1.0:1788201706.624325:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.624327:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.624328:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477765632, 477769728) 0 0 00000080:00200000:1.0:1788201706.624329:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.624332:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116642, 116642] 00020000:00200000:1.0:1788201706.624334:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477765632 chunk: [477765632, 477769728) 477769728, 4096 00020000:00200000:1.0:1788201706.624335:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.624342:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116642, 116642] 00000008:00010000:1.0:1788201706.624345:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.624346:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.624351:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624357:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477765632 477769727) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624362:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624367:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624371:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.624375:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477765632, 477769728) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.624377:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116642 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.624381:0:17463:0:(rw26.c:898:ll_write_end()) pos 477765632, len 4096, copied 4096 00000080:00200000:1.0:1788201706.624383:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c9a820e6[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116642] 00000080:00200000:1.0:1788201706.624385:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006fc77b4f fffffc0001029 2:0 ffff93c0c121dd20 116642 lru 00000080:00200000:1.0:1788201706.624389:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006a29862e 116642: 1< 2 - - > 2< 477765632 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68227 - -> 00000080:00200000:1.0:1788201706.624390:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c9a820e6 00000080:00200000:1.0:1788201706.624391:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.624392:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.624398:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.624405:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.624407:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.624408:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.624414:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.624415:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.624417:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.624419:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624424:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.624429:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.624430:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.624432:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.624434:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116642, 116642] 00000080:00200000:1.0:1788201706.624441:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.624443:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.624460:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.624462:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.624472:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477769728, count: 4096 00000080:00200000:1.0:1788201706.624478:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477769728, bytes: 4096 00000080:00200000:1.0:1788201706.624480:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.624482:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.624483:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477769728, 477773824) 0 0 00000080:00200000:1.0:1788201706.624484:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.624488:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116643, 116643] 00020000:00200000:1.0:1788201706.624489:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477769728 chunk: [477769728, 477773824) 477773824, 4096 00020000:00200000:1.0:1788201706.624491:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.624497:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116643, 116643] 00000008:00010000:1.0:1788201706.624500:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.624501:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.624506:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624512:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477769728 477773823) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624517:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624522:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624525:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.624529:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477769728, 477773824) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.624532:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116643 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.624536:0:17463:0:(rw26.c:898:ll_write_end()) pos 477769728, len 4096, copied 4096 00000080:00200000:1.0:1788201706.624538:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000805e59f6[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116643] 00000080:00200000:1.0:1788201706.624540:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d913a657 fffffc0001029 2:0 ffff93c0c121d9a0 116643 lru 00000080:00200000:1.0:1788201706.624543:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a7f54801 116643: 1< 2 - - > 2< 477769728 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68228 - -> 00000080:00200000:1.0:1788201706.624545:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000805e59f6 00000080:00200000:1.0:1788201706.624546:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.624547:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.624552:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.624560:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.624562:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.624563:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.624568:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.624570:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.624572:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.624574:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624578:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.624583:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.624584:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.624587:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.624588:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116643, 116643] 00000080:00200000:1.0:1788201706.624595:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.624597:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.624614:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.624617:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.624627:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477773824, count: 4096 00000080:00200000:1.0:1788201706.624632:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477773824, bytes: 4096 00000080:00200000:1.0:1788201706.624634:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.624637:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.624638:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477773824, 477777920) 0 0 00000080:00200000:1.0:1788201706.624639:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.624642:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116644, 116644] 00020000:00200000:1.0:1788201706.624644:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477773824 chunk: [477773824, 477777920) 477777920, 4096 00020000:00200000:1.0:1788201706.624645:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.624651:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116644, 116644] 00000008:00010000:1.0:1788201706.624654:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.624656:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.624661:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624666:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477773824 477777919) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624672:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624676:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624680:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.624684:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477773824, 477777920) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.624687:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116644 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.624690:0:17463:0:(rw26.c:898:ll_write_end()) pos 477773824, len 4096, copied 4096 00000080:00200000:1.0:1788201706.624693:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000070279ab[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116644] 00000080:00200000:1.0:1788201706.624694:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c24ed052 fffffc0001029 2:0 ffff93c0c121db60 116644 lru 00000080:00200000:1.0:1788201706.624698:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000072506526 116644: 1< 2 - - > 2< 477773824 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68229 - -> 00000080:00200000:1.0:1788201706.624700:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000070279ab 00000080:00200000:1.0:1788201706.624700:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.624702:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.624707:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.624714:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.624716:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.624717:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.624723:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.624724:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.624726:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.624728:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624732:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.624737:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.624738:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.624741:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.624743:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116644, 116644] 00000080:00200000:1.0:1788201706.624749:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.624751:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.624769:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.624771:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.624781:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477777920, count: 4096 00000080:00200000:1.0:1788201706.624787:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477777920, bytes: 4096 00000080:00200000:1.0:1788201706.624788:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.624791:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.624792:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477777920, 477782016) 0 0 00000080:00200000:1.0:1788201706.624793:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.624796:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116645, 116645] 00020000:00200000:1.0:1788201706.624798:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477777920 chunk: [477777920, 477782016) 477782016, 4096 00020000:00200000:1.0:1788201706.624800:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.624806:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116645, 116645] 00000008:00010000:1.0:1788201706.624809:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.624810:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.624815:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624820:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477777920 477782015) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624826:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624842:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624847:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.624855:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477777920, 477782016) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.624858:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116645 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.624862:0:17463:0:(rw26.c:898:ll_write_end()) pos 477777920, len 4096, copied 4096 00000080:00200000:1.0:1788201706.624864:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ddd4da5f[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116645] 00000080:00200000:1.0:1788201706.624866:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003bd4d2b2 fffffc0001029 2:0 ffff93c0c121dc40 116645 lru 00000080:00200000:1.0:1788201706.624869:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000072074af5 116645: 1< 2 - - > 2< 477777920 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68230 - -> 00000080:00200000:1.0:1788201706.624871:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ddd4da5f 00000080:00200000:1.0:1788201706.624872:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.624873:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.624878:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.624886:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.624888:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.624889:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.624895:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.624897:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.624898:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.624900:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624905:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.624909:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.624911:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.624914:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.624915:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116645, 116645] 00000080:00200000:1.0:1788201706.624922:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.624924:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.624941:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.624943:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.624954:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477782016, count: 4096 00000080:00200000:1.0:1788201706.624959:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477782016, bytes: 4096 00000080:00200000:1.0:1788201706.624961:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.624963:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.624964:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477782016, 477786112) 0 0 00000080:00200000:1.0:1788201706.624965:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.624968:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116646, 116646] 00020000:00200000:1.0:1788201706.624970:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477782016 chunk: [477782016, 477786112) 477786112, 4096 00020000:00200000:1.0:1788201706.624972:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.624978:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116646, 116646] 00000008:00010000:1.0:1788201706.624981:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.624982:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.624987:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624993:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477782016 477786111) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.624998:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625003:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625010:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.625026:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477782016, 477786112) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.625030:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116646 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.625034:0:17463:0:(rw26.c:898:ll_write_end()) pos 477782016, len 4096, copied 4096 00000080:00200000:1.0:1788201706.625037:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006ee62400[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116646] 00000080:00200000:1.0:1788201706.625038:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e6810a7b fffffc0001029 2:0 ffff93c0c121d540 116646 lru 00000080:00200000:1.0:1788201706.625042:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000087cc50e 116646: 1< 2 - - > 2< 477782016 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68231 - -> 00000080:00200000:1.0:1788201706.625043:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006ee62400 00000080:00200000:1.0:1788201706.625044:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.625045:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.625051:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.625059:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.625061:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.625062:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.625068:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.625070:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.625071:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.625074:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625079:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.625087:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.625088:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.625091:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.625093:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116646, 116646] 00000080:00200000:1.0:1788201706.625100:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.625102:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.625120:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.625122:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.625132:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477786112, count: 4096 00000080:00200000:1.0:1788201706.625138:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477786112, bytes: 4096 00000080:00200000:1.0:1788201706.625140:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.625142:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.625143:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477786112, 477790208) 0 0 00000080:00200000:1.0:1788201706.625145:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.625148:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116647, 116647] 00020000:00200000:1.0:1788201706.625150:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477786112 chunk: [477786112, 477790208) 477790208, 4096 00020000:00200000:1.0:1788201706.625152:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.625158:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116647, 116647] 00000008:00010000:1.0:1788201706.625161:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.625162:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.625167:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625173:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477786112 477790207) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625178:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625186:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625190:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.625194:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477786112, 477790208) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.625197:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116647 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.625202:0:17463:0:(rw26.c:898:ll_write_end()) pos 477786112, len 4096, copied 4096 00000080:00200000:1.0:1788201706.625206:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000069ca62ba[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116647] 00000080:00200000:1.0:1788201706.625208:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001066f62f fffffc0001029 2:0 ffff93c0c121d460 116647 lru 00000080:00200000:1.0:1788201706.625214:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000028a994ca 116647: 1< 2 - - > 2< 477786112 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68232 - -> 00000080:00200000:1.0:1788201706.625216:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000069ca62ba 00000080:00200000:1.0:1788201706.625216:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.625218:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.625227:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.625238:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.625241:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.625243:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.625251:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.625253:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.625255:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.625259:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625269:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.625274:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.625275:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.625278:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.625280:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116647, 116647] 00000080:00200000:1.0:1788201706.625287:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.625289:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.625307:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.625309:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.625319:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477790208, count: 4096 00000080:00200000:1.0:1788201706.625325:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477790208, bytes: 4096 00000080:00200000:1.0:1788201706.625327:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.625329:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.625330:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477790208, 477794304) 0 0 00000080:00200000:1.0:1788201706.625332:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.625335:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116648, 116648] 00020000:00200000:1.0:1788201706.625337:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477790208 chunk: [477790208, 477794304) 477794304, 4096 00020000:00200000:1.0:1788201706.625338:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.625345:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116648, 116648] 00000008:00010000:1.0:1788201706.625348:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.625349:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.625354:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625360:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477790208 477794303) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625369:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625373:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625377:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.625381:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477790208, 477794304) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.625384:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116648 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.625388:0:17463:0:(rw26.c:898:ll_write_end()) pos 477790208, len 4096, copied 4096 00000080:00200000:1.0:1788201706.625390:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000090fe0f84[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116648] 00000080:00200000:1.0:1788201706.625392:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006eaba6ff fffffc0001029 2:0 ffff93c0c121d0e0 116648 lru 00000080:00200000:1.0:1788201706.625396:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000081e210f6 116648: 1< 2 - - > 2< 477790208 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68233 - -> 00000080:00200000:1.0:1788201706.625397:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000090fe0f84 00000080:00200000:1.0:1788201706.625398:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.625399:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.625405:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.625412:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.625414:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.625415:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.625421:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.625423:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.625425:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.625427:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625432:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.625436:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.625438:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.625440:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.625442:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116648, 116648] 00000080:00200000:1.0:1788201706.625449:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.625451:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.625468:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.625470:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.625480:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477794304, count: 4096 00000080:00200000:1.0:1788201706.625486:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477794304, bytes: 4096 00000080:00200000:1.0:1788201706.625487:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.625490:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.625491:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477794304, 477798400) 0 0 00000080:00200000:1.0:1788201706.625492:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.625495:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116649, 116649] 00020000:00200000:1.0:1788201706.625497:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477794304 chunk: [477794304, 477798400) 477798400, 4096 00020000:00200000:1.0:1788201706.625498:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.625505:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116649, 116649] 00000008:00010000:1.0:1788201706.625508:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.625509:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.625514:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625523:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477794304 477798399) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625528:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625532:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625536:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.625540:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477794304, 477798400) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.625543:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116649 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.625546:0:17463:0:(rw26.c:898:ll_write_end()) pos 477794304, len 4096, copied 4096 00000080:00200000:1.0:1788201706.625548:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000021cfe2d2[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116649] 00000080:00200000:1.0:1788201706.625550:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a321ead1 fffffc0001029 2:0 ffff93c0c121d700 116649 lru 00000080:00200000:1.0:1788201706.625553:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003117a0f3 116649: 1< 2 - - > 2< 477794304 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68234 - -> 00000080:00200000:1.0:1788201706.625555:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000021cfe2d2 00000080:00200000:1.0:1788201706.625555:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.625556:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.625561:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.625569:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.625570:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.625572:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.625577:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.625579:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.625581:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.625583:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625588:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.625593:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.625594:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.625596:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.625598:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116649, 116649] 00000080:00200000:1.0:1788201706.625605:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.625607:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.625624:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.625626:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.625636:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477798400, count: 4096 00000080:00200000:1.0:1788201706.625642:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477798400, bytes: 4096 00000080:00200000:1.0:1788201706.625644:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.625646:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.625647:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477798400, 477802496) 0 0 00000080:00200000:1.0:1788201706.625648:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.625651:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116650, 116650] 00020000:00200000:1.0:1788201706.625653:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477798400 chunk: [477798400, 477802496) 477802496, 4096 00020000:00200000:1.0:1788201706.625655:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.625661:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116650, 116650] 00000008:00010000:1.0:1788201706.625664:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.625665:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.625670:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625676:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477798400 477802495) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625682:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625686:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625690:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.625694:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477798400, 477802496) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.625697:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116650 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.625700:0:17463:0:(rw26.c:898:ll_write_end()) pos 477798400, len 4096, copied 4096 00000080:00200000:1.0:1788201706.625702:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009418d18c[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116650] 00000080:00200000:1.0:1788201706.625704:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cdeec2d4 fffffc0001029 2:0 ffff93c0c121d000 116650 lru 00000080:00200000:1.0:1788201706.625707:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f4e9f06a 116650: 1< 2 - - > 2< 477798400 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68235 - -> 00000080:00200000:1.0:1788201706.625709:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009418d18c 00000080:00200000:1.0:1788201706.625710:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.625711:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.625716:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.625724:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.625725:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.625727:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.625733:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.625734:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.625736:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.625738:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625743:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.625747:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.625749:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.625751:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.625753:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116650, 116650] 00000080:00200000:1.0:1788201706.625760:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.625762:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.625779:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.625781:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.625791:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477802496, count: 4096 00000080:00200000:1.0:1788201706.625797:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477802496, bytes: 4096 00000080:00200000:1.0:1788201706.625799:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.625801:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.625802:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477802496, 477806592) 0 0 00000080:00200000:1.0:1788201706.625804:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.625807:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116651, 116651] 00020000:00200000:1.0:1788201706.625809:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477802496 chunk: [477802496, 477806592) 477806592, 4096 00020000:00200000:1.0:1788201706.625810:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.625816:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116651, 116651] 00000008:00010000:1.0:1788201706.625819:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.625821:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.625825:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625844:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477802496 477806591) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625849:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625853:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625857:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.625862:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477802496, 477806592) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.625866:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116651 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.625869:0:17463:0:(rw26.c:898:ll_write_end()) pos 477802496, len 4096, copied 4096 00000080:00200000:1.0:1788201706.625872:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c96411d6[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116651] 00000080:00200000:1.0:1788201706.625873:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a8dc86b5 fffffc0001029 2:0 ffff93c0c121da80 116651 lru 00000080:00200000:1.0:1788201706.625877:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001538e7b2 116651: 1< 2 - - > 2< 477802496 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68236 - -> 00000080:00200000:1.0:1788201706.625878:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c96411d6 00000080:00200000:1.0:1788201706.625879:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.625880:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.625886:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.625897:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.625898:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.625900:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.625906:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.625907:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.625909:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.625911:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.625916:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.625921:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.625923:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.625925:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.625927:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116651, 116651] 00000080:00200000:1.0:1788201706.625933:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.625935:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.625953:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.625955:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.625965:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477806592, count: 4096 00000080:00200000:1.0:1788201706.625971:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477806592, bytes: 4096 00000080:00200000:1.0:1788201706.625973:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.625975:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.625976:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477806592, 477810688) 0 0 00000080:00200000:1.0:1788201706.625977:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.625980:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116652, 116652] 00020000:00200000:1.0:1788201706.625982:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477806592 chunk: [477806592, 477810688) 477810688, 4096 00020000:00200000:1.0:1788201706.625984:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.625990:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116652, 116652] 00000008:00010000:1.0:1788201706.625993:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.625994:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.625999:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626005:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477806592 477810687) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626010:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626026:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626031:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.626037:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477806592, 477810688) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.626040:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116652 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.626044:0:17463:0:(rw26.c:898:ll_write_end()) pos 477806592, len 4096, copied 4096 00000080:00200000:1.0:1788201706.626046:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ea8a276a[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116652] 00000080:00200000:1.0:1788201706.626048:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c0050c85 fffffc0001029 2:0 ffff93c0c121e000 116652 lru 00000080:00200000:1.0:1788201706.626051:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003cb54e0b 116652: 1< 2 - - > 2< 477806592 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68237 - -> 00000080:00200000:1.0:1788201706.626053:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ea8a276a 00000080:00200000:1.0:1788201706.626054:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.626055:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.626061:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.626069:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.626071:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.626072:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.626078:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.626080:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.626081:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.626084:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626089:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.626094:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.626095:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.626098:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.626099:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116652, 116652] 00000080:00200000:1.0:1788201706.626106:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.626108:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.626126:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.626128:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.626138:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477810688, count: 4096 00000080:00200000:1.0:1788201706.626144:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477810688, bytes: 4096 00000080:00200000:1.0:1788201706.626146:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.626148:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.626149:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477810688, 477814784) 0 0 00000080:00200000:1.0:1788201706.626151:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.626154:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116653, 116653] 00020000:00200000:1.0:1788201706.626156:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477810688 chunk: [477810688, 477814784) 477814784, 4096 00020000:00200000:1.0:1788201706.626157:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.626164:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116653, 116653] 00000008:00010000:1.0:1788201706.626167:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.626168:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.626173:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626179:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477810688 477814783) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626185:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626189:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626193:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.626197:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477810688, 477814784) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.626202:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116653 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.626206:0:17463:0:(rw26.c:898:ll_write_end()) pos 477810688, len 4096, copied 4096 00000080:00200000:1.0:1788201706.626210:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000351fd506[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116653] 00000080:00200000:1.0:1788201706.626213:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005dd7055d fffffc0001029 2:0 ffff93c0c121ea80 116653 lru 00000080:00200000:1.0:1788201706.626218:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a4a4c2db 116653: 1< 2 - - > 2< 477810688 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68238 - -> 00000080:00200000:1.0:1788201706.626221:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000351fd506 00000080:00200000:1.0:1788201706.626223:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.626224:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.626233:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.626244:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.626247:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.626248:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.626255:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.626256:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.626258:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.626261:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626266:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.626271:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.626272:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.626274:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.626276:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116653, 116653] 00000080:00200000:1.0:1788201706.626283:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.626286:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.626304:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.626306:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.626316:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477814784, count: 4096 00000080:00200000:1.0:1788201706.626322:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477814784, bytes: 4096 00000080:00200000:1.0:1788201706.626324:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.626326:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.626327:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477814784, 477818880) 0 0 00000080:00200000:1.0:1788201706.626329:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.626332:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116654, 116654] 00020000:00200000:1.0:1788201706.626334:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477814784 chunk: [477814784, 477818880) 477818880, 4096 00020000:00200000:1.0:1788201706.626335:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.626341:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116654, 116654] 00000008:00010000:1.0:1788201706.626344:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.626346:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.626351:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626356:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477814784 477818879) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626362:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626366:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626370:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.626374:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477814784, 477818880) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.626377:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116654 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.626381:0:17463:0:(rw26.c:898:ll_write_end()) pos 477814784, len 4096, copied 4096 00000080:00200000:1.0:1788201706.626383:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000556c838a[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116654] 00000080:00200000:1.0:1788201706.626385:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b6d26288 fffffc0001029 2:0 ffff93c0c121e380 116654 lru 00000080:00200000:1.0:1788201706.626389:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000019625234 116654: 1< 2 - - > 2< 477814784 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68239 - -> 00000080:00200000:1.0:1788201706.626390:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000556c838a 00000080:00200000:1.0:1788201706.626391:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.626392:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.626397:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.626404:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.626406:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.626407:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.626413:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.626415:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.626416:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.626419:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626423:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.626428:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.626430:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.626432:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.626434:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116654, 116654] 00000080:00200000:1.0:1788201706.626441:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.626442:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.626460:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.626462:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.626472:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477818880, count: 4096 00000080:00200000:1.0:1788201706.626478:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477818880, bytes: 4096 00000080:00200000:1.0:1788201706.626480:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.626482:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.626483:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477818880, 477822976) 0 0 00000080:00200000:1.0:1788201706.626484:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.626487:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116655, 116655] 00020000:00200000:1.0:1788201706.626489:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477818880 chunk: [477818880, 477822976) 477822976, 4096 00020000:00200000:1.0:1788201706.626490:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.626497:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116655, 116655] 00000008:00010000:1.0:1788201706.626500:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.626501:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.626506:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626512:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477818880 477822975) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626517:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626521:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626525:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.626530:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477818880, 477822976) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.626533:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116655 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.626537:0:17463:0:(rw26.c:898:ll_write_end()) pos 477818880, len 4096, copied 4096 00000080:00200000:1.0:1788201706.626539:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000083a0438a[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116655] 00000080:00200000:1.0:1788201706.626541:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d1b17195 fffffc0001029 2:0 ffff93c0c121e8c0 116655 lru 00000080:00200000:1.0:1788201706.626544:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d7a08583 116655: 1< 2 - - > 2< 477818880 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68240 - -> 00000080:00200000:1.0:1788201706.626546:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000083a0438a 00000080:00200000:1.0:1788201706.626547:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.626548:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.626553:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.626561:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.626562:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.626564:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.626569:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.626571:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.626572:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.626575:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626579:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.626583:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.626585:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.626587:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.626589:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116655, 116655] 00000080:00200000:1.0:1788201706.626596:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.626598:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.626615:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.626617:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.626627:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477822976, count: 4096 00000080:00200000:1.0:1788201706.626633:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477822976, bytes: 4096 00000080:00200000:1.0:1788201706.626635:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.626637:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.626638:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477822976, 477827072) 0 0 00000080:00200000:1.0:1788201706.626639:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.626642:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116656, 116656] 00020000:00200000:1.0:1788201706.626644:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477822976 chunk: [477822976, 477827072) 477827072, 4096 00020000:00200000:1.0:1788201706.626646:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.626652:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116656, 116656] 00000008:00010000:1.0:1788201706.626655:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.626657:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.626662:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626667:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477822976 477827071) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626673:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626677:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626681:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.626685:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477822976, 477827072) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.626689:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116656 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.626692:0:17463:0:(rw26.c:898:ll_write_end()) pos 477822976, len 4096, copied 4096 00000080:00200000:1.0:1788201706.626694:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f418c4a6[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116656] 00000080:00200000:1.0:1788201706.626696:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008a7a358e fffffc0001029 2:0 ffff93c0c121eee0 116656 lru 00000080:00200000:1.0:1788201706.626700:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000083c66ade 116656: 1< 2 - - > 2< 477822976 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68241 - -> 00000080:00200000:1.0:1788201706.626702:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f418c4a6 00000080:00200000:1.0:1788201706.626702:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.626704:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.626709:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.626717:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.626719:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.626720:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.626726:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.626728:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.626729:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.626732:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626737:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.626741:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.626743:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.626745:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.626747:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116656, 116656] 00000080:00200000:1.0:1788201706.626753:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.626755:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.626773:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.626775:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.626785:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477827072, count: 4096 00000080:00200000:1.0:1788201706.626791:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477827072, bytes: 4096 00000080:00200000:1.0:1788201706.626793:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.626795:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.626796:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477827072, 477831168) 0 0 00000080:00200000:1.0:1788201706.626797:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.626800:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116657, 116657] 00020000:00200000:1.0:1788201706.626802:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477827072 chunk: [477827072, 477831168) 477831168, 4096 00020000:00200000:1.0:1788201706.626804:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.626810:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116657, 116657] 00000008:00010000:1.0:1788201706.626813:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.626814:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.626819:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626825:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477827072 477831167) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626843:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626848:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626852:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.626860:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477827072, 477831168) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.626863:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116657 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.626867:0:17463:0:(rw26.c:898:ll_write_end()) pos 477827072, len 4096, copied 4096 00000080:00200000:1.0:1788201706.626869:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000002dd3364[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116657] 00000080:00200000:1.0:1788201706.626871:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b1b04ba4 fffffc0001029 2:0 ffff93c0c121e7e0 116657 lru 00000080:00200000:1.0:1788201706.626874:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005efb6713 116657: 1< 2 - - > 2< 477827072 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68242 - -> 00000080:00200000:1.0:1788201706.626876:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000002dd3364 00000080:00200000:1.0:1788201706.626877:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.626878:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.626883:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.626891:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.626892:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.626894:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.626899:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.626901:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.626903:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.626905:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626910:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.626915:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.626916:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.626919:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.626921:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116657, 116657] 00000080:00200000:1.0:1788201706.626927:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.626930:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.626947:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.626949:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.626959:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477831168, count: 4096 00000080:00200000:1.0:1788201706.626965:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477831168, bytes: 4096 00000080:00200000:1.0:1788201706.626967:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.626969:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.626970:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477831168, 477835264) 0 0 00000080:00200000:1.0:1788201706.626971:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.626975:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116658, 116658] 00020000:00200000:1.0:1788201706.626977:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477831168 chunk: [477831168, 477835264) 477835264, 4096 00020000:00200000:1.0:1788201706.626978:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.626984:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116658, 116658] 00000008:00010000:1.0:1788201706.626987:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.626988:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.626993:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.626999:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477831168 477835263) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627005:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627009:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627032:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.627038:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477831168, 477835264) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.627042:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116658 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.627062:0:17463:0:(rw26.c:898:ll_write_end()) pos 477831168, len 4096, copied 4096 00000080:00200000:1.0:1788201706.627066:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e8fd7dbf[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116658] 00000080:00200000:1.0:1788201706.627067:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006e918ed6 fffffc0001029 2:0 ffff93c0c121e2a0 116658 lru 00000080:00200000:1.0:1788201706.627071:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007fbee1b9 116658: 1< 2 - - > 2< 477831168 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68243 - -> 00000080:00200000:1.0:1788201706.627073:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e8fd7dbf 00000080:00200000:1.0:1788201706.627074:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.627075:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.627081:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.627088:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.627090:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.627092:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.627098:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.627099:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.627101:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.627103:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627108:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.627117:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.627118:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.627121:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.627123:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116658, 116658] 00000080:00200000:1.0:1788201706.627130:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.627132:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.627150:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.627152:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.627163:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477835264, count: 4096 00000080:00200000:1.0:1788201706.627168:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477835264, bytes: 4096 00000080:00200000:1.0:1788201706.627170:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.627172:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.627173:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477835264, 477839360) 0 0 00000080:00200000:1.0:1788201706.627175:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.627178:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116659, 116659] 00020000:00200000:1.0:1788201706.627180:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477835264 chunk: [477835264, 477839360) 477839360, 4096 00020000:00200000:1.0:1788201706.627181:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.627188:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116659, 116659] 00000008:00010000:1.0:1788201706.627191:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.627192:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.627198:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627207:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477835264 477839359) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627216:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627228:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627236:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.627243:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477835264, 477839360) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.627248:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116659 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.627253:0:17463:0:(rw26.c:898:ll_write_end()) pos 477835264, len 4096, copied 4096 00000080:00200000:1.0:1788201706.627257:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000afbcb172[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116659] 00000080:00200000:1.0:1788201706.627259:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007d6632d7 fffffc0001029 2:0 ffff93c0c121ee00 116659 lru 00000080:00200000:1.0:1788201706.627264:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002b735000 116659: 1< 2 - - > 2< 477835264 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68244 - -> 00000080:00200000:1.0:1788201706.627266:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000afbcb172 00000080:00200000:1.0:1788201706.627267:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.627269:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.627277:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.627287:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.627289:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.627291:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.627299:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.627301:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.627304:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.627308:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627319:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.627327:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.627330:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.627334:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.627337:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116659, 116659] 00000080:00200000:1.0:1788201706.627347:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.627350:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.627378:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.627381:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.627396:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477839360, count: 4096 00000080:00200000:1.0:1788201706.627406:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477839360, bytes: 4096 00000080:00200000:1.0:1788201706.627409:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.627413:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.627414:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477839360, 477843456) 0 0 00000080:00200000:1.0:1788201706.627417:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.627420:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116660, 116660] 00020000:00200000:1.0:1788201706.627422:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477839360 chunk: [477839360, 477843456) 477843456, 4096 00020000:00200000:1.0:1788201706.627424:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.627432:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116660, 116660] 00000008:00010000:1.0:1788201706.627435:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.627437:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.627442:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627448:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477839360 477843455) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627457:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627462:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627466:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.627470:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477839360, 477843456) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.627473:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116660 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.627476:0:17463:0:(rw26.c:898:ll_write_end()) pos 477839360, len 4096, copied 4096 00000080:00200000:1.0:1788201706.627479:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a52ca655[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116660] 00000080:00200000:1.0:1788201706.627480:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000007dc504d fffffc0001029 2:0 ffff93c0c121e1c0 116660 lru 00000080:00200000:1.0:1788201706.627484:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c5341c40 116660: 1< 2 - - > 2< 477839360 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68245 - -> 00000080:00200000:1.0:1788201706.627485:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a52ca655 00000080:00200000:1.0:1788201706.627486:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.627487:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.627493:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.627501:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.627502:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.627504:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.627510:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.627511:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.627513:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.627516:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627521:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.627526:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.627527:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.627529:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.627531:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116660, 116660] 00000080:00200000:1.0:1788201706.627538:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.627540:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.627558:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.627560:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.627571:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477843456, count: 4096 00000080:00200000:1.0:1788201706.627577:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477843456, bytes: 4096 00000080:00200000:1.0:1788201706.627578:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.627580:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.627581:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477843456, 477847552) 0 0 00000080:00200000:1.0:1788201706.627583:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.627586:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116661, 116661] 00020000:00200000:1.0:1788201706.627588:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477843456 chunk: [477843456, 477847552) 477847552, 4096 00020000:00200000:1.0:1788201706.627589:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.627596:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116661, 116661] 00000008:00010000:1.0:1788201706.627598:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.627600:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.627605:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627614:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477843456 477847551) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627618:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627623:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627627:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.627631:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477843456, 477847552) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.627634:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116661 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.627637:0:17463:0:(rw26.c:898:ll_write_end()) pos 477843456, len 4096, copied 4096 00000080:00200000:1.0:1788201706.627640:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000025f2025[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116661] 00000080:00200000:1.0:1788201706.627642:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000189bb35 fffffc0001029 2:0 ffff93c0c121e620 116661 lru 00000080:00200000:1.0:1788201706.627645:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b9f6239f 116661: 1< 2 - - > 2< 477843456 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68246 - -> 00000080:00200000:1.0:1788201706.627647:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000025f2025 00000080:00200000:1.0:1788201706.627648:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.627649:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.627654:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.627662:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.627663:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.627665:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.627671:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.627673:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.627674:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.627676:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627681:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.627686:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.627688:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.627690:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.627691:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116661, 116661] 00000080:00200000:1.0:1788201706.627698:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.627700:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.627717:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.627719:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.627729:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477847552, count: 4096 00000080:00200000:1.0:1788201706.627735:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477847552, bytes: 4096 00000080:00200000:1.0:1788201706.627736:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.627739:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.627740:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477847552, 477851648) 0 0 00000080:00200000:1.0:1788201706.627741:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.627744:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116662, 116662] 00020000:00200000:1.0:1788201706.627746:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477847552 chunk: [477847552, 477851648) 477851648, 4096 00020000:00200000:1.0:1788201706.627748:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.627754:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116662, 116662] 00000008:00010000:1.0:1788201706.627759:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.627761:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.627768:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627777:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477847552 477851647) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627786:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627793:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627801:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.627807:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477847552, 477851648) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.627811:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116662 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.627815:0:17463:0:(rw26.c:898:ll_write_end()) pos 477847552, len 4096, copied 4096 00000080:00200000:1.0:1788201706.627817:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000faf22782[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116662] 00000080:00200000:1.0:1788201706.627819:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a27c3101 fffffc0001029 2:0 ffff93c0c121ed20 116662 lru 00000080:00200000:1.0:1788201706.627823:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000059a88fba 116662: 1< 2 - - > 2< 477847552 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68247 - -> 00000080:00200000:1.0:1788201706.627824:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000faf22782 00000080:00200000:1.0:1788201706.627825:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.627826:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.627846:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.627854:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.627856:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.627858:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.627864:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.627866:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.627868:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.627870:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627875:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.627880:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.627881:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.627884:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.627886:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116662, 116662] 00000080:00200000:1.0:1788201706.627893:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.627895:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.627913:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.627915:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.627925:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477851648, count: 4096 00000080:00200000:1.0:1788201706.627931:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477851648, bytes: 4096 00000080:00200000:1.0:1788201706.627933:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.627935:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.627936:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477851648, 477855744) 0 0 00000080:00200000:1.0:1788201706.627938:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.627941:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116663, 116663] 00020000:00200000:1.0:1788201706.627943:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477851648 chunk: [477851648, 477855744) 477855744, 4096 00020000:00200000:1.0:1788201706.627944:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.627950:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116663, 116663] 00000008:00010000:1.0:1788201706.627954:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.627955:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.627960:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627966:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477851648 477855743) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627971:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627975:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.627979:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.627983:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477851648, 477855744) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.627986:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116663 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.627989:0:17463:0:(rw26.c:898:ll_write_end()) pos 477851648, len 4096, copied 4096 00000080:00200000:1.0:1788201706.627992:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e594f958[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116663] 00000080:00200000:1.0:1788201706.627993:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a17e2322 fffffc0001029 2:0 ffff93c0c121e9a0 116663 lru 00000080:00200000:1.0:1788201706.627997:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006d6c9950 116663: 1< 2 - - > 2< 477851648 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68248 - -> 00000080:00200000:1.0:1788201706.627998:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e594f958 00000080:00200000:1.0:1788201706.627999:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.628000:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.628005:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.628029:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.628032:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.628033:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.628040:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.628042:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.628044:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.628047:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628052:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.628057:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.628058:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.628061:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.628063:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116663, 116663] 00000080:00200000:1.0:1788201706.628070:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.628071:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.628090:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.628092:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.628102:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477855744, count: 4096 00000080:00200000:1.0:1788201706.628108:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477855744, bytes: 4096 00000080:00200000:1.0:1788201706.628110:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.628112:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.628113:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477855744, 477859840) 0 0 00000080:00200000:1.0:1788201706.628114:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.628118:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116664, 116664] 00020000:00200000:1.0:1788201706.628120:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477855744 chunk: [477855744, 477859840) 477859840, 4096 00020000:00200000:1.0:1788201706.628122:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.628128:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116664, 116664] 00000008:00010000:1.0:1788201706.628131:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.628132:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.628137:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628143:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477855744 477859839) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628148:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628153:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628157:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.628161:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477855744, 477859840) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.628163:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116664 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.628167:0:17463:0:(rw26.c:898:ll_write_end()) pos 477855744, len 4096, copied 4096 00000080:00200000:1.0:1788201706.628169:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009134b836[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116664] 00000080:00200000:1.0:1788201706.628171:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000051370f19 fffffc0001029 2:0 ffff93c0c121eb60 116664 lru 00000080:00200000:1.0:1788201706.628174:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c6469746 116664: 1< 2 - - > 2< 477855744 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68249 - -> 00000080:00200000:1.0:1788201706.628175:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009134b836 00000080:00200000:1.0:1788201706.628176:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.628177:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.628183:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.628191:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.628192:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.628193:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.628201:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.628204:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.628206:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.628210:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628218:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.628227:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.628229:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.628233:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.628236:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116664, 116664] 00000080:00200000:1.0:1788201706.628247:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.628250:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.628269:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.628271:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.628282:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477859840, count: 4096 00000080:00200000:1.0:1788201706.628288:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477859840, bytes: 4096 00000080:00200000:1.0:1788201706.628290:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.628292:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.628293:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477859840, 477863936) 0 0 00000080:00200000:1.0:1788201706.628295:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.628298:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116665, 116665] 00020000:00200000:1.0:1788201706.628300:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477859840 chunk: [477859840, 477863936) 477863936, 4096 00020000:00200000:1.0:1788201706.628301:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.628308:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116665, 116665] 00000008:00010000:1.0:1788201706.628311:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.628313:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.628318:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628323:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477859840 477863935) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628329:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628334:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628338:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.628342:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477859840, 477863936) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.628345:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116665 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.628349:0:17463:0:(rw26.c:898:ll_write_end()) pos 477859840, len 4096, copied 4096 00000080:00200000:1.0:1788201706.628351:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a923d94e[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116665] 00000080:00200000:1.0:1788201706.628353:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d04ff26e fffffc0001029 2:0 ffff93c0c121ec40 116665 lru 00000080:00200000:1.0:1788201706.628357:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a6aae9f2 116665: 1< 2 - - > 2< 477859840 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68250 - -> 00000080:00200000:1.0:1788201706.628360:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a923d94e 00000080:00200000:1.0:1788201706.628360:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.628362:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.628368:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.628375:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.628377:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.628378:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.628384:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.628386:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.628388:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.628390:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628395:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.628399:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.628401:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.628403:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.628405:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116665, 116665] 00000080:00200000:1.0:1788201706.628411:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.628413:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.628431:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.628433:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.628443:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477863936, count: 4096 00000080:00200000:1.0:1788201706.628448:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477863936, bytes: 4096 00000080:00200000:1.0:1788201706.628451:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.628453:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.628454:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477863936, 477868032) 0 0 00000080:00200000:1.0:1788201706.628455:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.628458:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116666, 116666] 00020000:00200000:1.0:1788201706.628460:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477863936 chunk: [477863936, 477868032) 477868032, 4096 00020000:00200000:1.0:1788201706.628461:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.628468:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116666, 116666] 00000008:00010000:1.0:1788201706.628471:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.628472:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.628477:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628483:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477863936 477868031) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628488:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628493:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628497:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.628501:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477863936, 477868032) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.628504:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116666 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.628508:0:17463:0:(rw26.c:898:ll_write_end()) pos 477863936, len 4096, copied 4096 00000080:00200000:1.0:1788201706.628510:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f5fc8a11[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116666] 00000080:00200000:1.0:1788201706.628513:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e1ab67d2 fffffc0001029 2:0 ffff93c0c121e540 116666 lru 00000080:00200000:1.0:1788201706.628516:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009584bc72 116666: 1< 2 - - > 2< 477863936 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68251 - -> 00000080:00200000:1.0:1788201706.628518:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f5fc8a11 00000080:00200000:1.0:1788201706.628518:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.628519:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.628525:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.628532:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.628534:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.628535:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.628541:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.628542:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.628544:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.628546:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628551:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.628556:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.628557:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.628559:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.628561:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116666, 116666] 00000080:00200000:1.0:1788201706.628568:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.628570:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.628587:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.628589:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.628599:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477868032, count: 4096 00000080:00200000:1.0:1788201706.628605:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477868032, bytes: 4096 00000080:00200000:1.0:1788201706.628607:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.628609:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.628610:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477868032, 477872128) 0 0 00000080:00200000:1.0:1788201706.628612:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.628615:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116667, 116667] 00020000:00200000:1.0:1788201706.628617:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477868032 chunk: [477868032, 477872128) 477872128, 4096 00020000:00200000:1.0:1788201706.628618:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.628624:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116667, 116667] 00000008:00010000:1.0:1788201706.628627:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.628628:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.628633:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628638:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477868032 477872127) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628644:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628648:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628652:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.628656:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477868032, 477872128) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.628659:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116667 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.628663:0:17463:0:(rw26.c:898:ll_write_end()) pos 477868032, len 4096, copied 4096 00000080:00200000:1.0:1788201706.628665:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004b3ecefd[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116667] 00000080:00200000:1.0:1788201706.628667:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d8f6884b fffffc0001029 2:0 ffff93c0c121e460 116667 lru 00000080:00200000:1.0:1788201706.628670:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001200f601 116667: 1< 2 - - > 2< 477868032 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68252 - -> 00000080:00200000:1.0:1788201706.628672:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004b3ecefd 00000080:00200000:1.0:1788201706.628672:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.628674:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.628679:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.628687:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.628688:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.628690:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.628695:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.628697:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.628698:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.628701:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628706:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.628711:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.628712:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.628714:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.628716:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116667, 116667] 00000080:00200000:1.0:1788201706.628722:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.628724:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.628744:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.628748:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.628763:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477872128, count: 4096 00000080:00200000:1.0:1788201706.628774:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477872128, bytes: 4096 00000080:00200000:1.0:1788201706.628777:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.628780:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.628782:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477872128, 477876224) 0 0 00000080:00200000:1.0:1788201706.628784:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.628789:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116668, 116668] 00020000:00200000:1.0:1788201706.628792:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477872128 chunk: [477872128, 477876224) 477876224, 4096 00020000:00200000:1.0:1788201706.628794:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.628805:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116668, 116668] 00000008:00010000:1.0:1788201706.628810:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.628812:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.628819:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628828:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477872128 477876223) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628851:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628856:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628861:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.628866:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477872128, 477876224) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.628870:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116668 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.628874:0:17463:0:(rw26.c:898:ll_write_end()) pos 477872128, len 4096, copied 4096 00000080:00200000:1.0:1788201706.628876:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d3361498[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116668] 00000080:00200000:1.0:1788201706.628878:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000f04a724 fffffc0001029 2:0 ffff93c0c121e0e0 116668 lru 00000080:00200000:1.0:1788201706.628882:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000eb5321f6 116668: 1< 2 - - > 2< 477872128 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68253 - -> 00000080:00200000:1.0:1788201706.628883:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d3361498 00000080:00200000:1.0:1788201706.628884:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.628885:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.628891:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.628899:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.628901:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.628902:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.628909:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.628911:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.628912:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.628915:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.628919:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.628924:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.628926:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.628929:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.628931:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116668, 116668] 00000080:00200000:1.0:1788201706.628938:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.628940:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.628958:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.628960:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.628971:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477876224, count: 4096 00000080:00200000:1.0:1788201706.628977:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477876224, bytes: 4096 00000080:00200000:1.0:1788201706.628978:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.628981:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.628982:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477876224, 477880320) 0 0 00000080:00200000:1.0:1788201706.628983:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.628986:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116669, 116669] 00020000:00200000:1.0:1788201706.628988:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477876224 chunk: [477876224, 477880320) 477880320, 4096 00020000:00200000:1.0:1788201706.628990:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.628996:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116669, 116669] 00000008:00010000:1.0:1788201706.628999:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.629000:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.629005:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629011:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477876224 477880319) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629031:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629036:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629040:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.629049:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477876224, 477880320) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.629052:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116669 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.629056:0:17463:0:(rw26.c:898:ll_write_end()) pos 477876224, len 4096, copied 4096 00000080:00200000:1.0:1788201706.629058:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000091c569a[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116669] 00000080:00200000:1.0:1788201706.629060:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000e8bd6d3 fffffc0001029 2:0 ffff93c0c121e700 116669 lru 00000080:00200000:1.0:1788201706.629064:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008a8de616 116669: 1< 2 - - > 2< 477876224 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68254 - -> 00000080:00200000:1.0:1788201706.629065:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000091c569a 00000080:00200000:1.0:1788201706.629066:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.629067:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.629073:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.629081:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.629083:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.629084:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.629090:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.629092:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.629094:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.629096:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629101:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.629106:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.629107:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.629110:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.629112:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116669, 116669] 00000080:00200000:1.0:1788201706.629119:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.629121:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.629139:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.629141:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.629151:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477880320, count: 4096 00000080:00200000:1.0:1788201706.629157:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477880320, bytes: 4096 00000080:00200000:1.0:1788201706.629159:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.629161:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.629162:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477880320, 477884416) 0 0 00000080:00200000:1.0:1788201706.629163:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.629166:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116670, 116670] 00020000:00200000:1.0:1788201706.629168:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477880320 chunk: [477880320, 477884416) 477884416, 4096 00020000:00200000:1.0:1788201706.629170:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.629176:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116670, 116670] 00000008:00010000:1.0:1788201706.629179:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.629181:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.629186:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629191:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477880320 477884415) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629197:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629204:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629216:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.629224:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477880320, 477884416) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.629229:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116670 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.629234:0:17463:0:(rw26.c:898:ll_write_end()) pos 477880320, len 4096, copied 4096 00000080:00200000:1.0:1788201706.629238:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003c84fd68[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116670] 00000080:00200000:1.0:1788201706.629241:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004df67adb fffffc0001029 2:0 ffff93c0c121fa80 116670 lru 00000080:00200000:1.0:1788201706.629246:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005bb736c2 116670: 1< 2 - - > 2< 477880320 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68255 - -> 00000080:00200000:1.0:1788201706.629248:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003c84fd68 00000080:00200000:1.0:1788201706.629248:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.629250:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.629256:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.629264:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.629266:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.629267:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.629273:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.629275:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.629276:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.629279:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629284:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.629293:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.629294:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.629297:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.629299:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116670, 116670] 00000080:00200000:1.0:1788201706.629306:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.629308:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.629326:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.629328:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.629338:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477884416, count: 4096 00000080:00200000:1.0:1788201706.629344:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477884416, bytes: 4096 00000080:00200000:1.0:1788201706.629346:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.629348:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.629349:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477884416, 477888512) 0 0 00000080:00200000:1.0:1788201706.629351:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.629354:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116671, 116671] 00020000:00200000:1.0:1788201706.629356:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477884416 chunk: [477884416, 477888512) 477888512, 4096 00020000:00200000:1.0:1788201706.629357:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.629363:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116671, 116671] 00000008:00010000:1.0:1788201706.629366:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.629368:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.629373:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629378:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477884416 477888511) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629384:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629392:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629396:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.629400:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477884416, 477888512) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.629403:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116671 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.629406:0:17463:0:(rw26.c:898:ll_write_end()) pos 477884416, len 4096, copied 4096 00000080:00200000:1.0:1788201706.629409:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005111af1f[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116671] 00000080:00200000:1.0:1788201706.629410:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e6ba22c7 fffffc0001029 2:0 ffff93c0c121f380 116671 lru 00000080:00200000:1.0:1788201706.629414:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000255ab3c0 116671: 1< 2 - - > 2< 477884416 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68256 - -> 00000080:00200000:1.0:1788201706.629415:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005111af1f 00000080:00200000:1.0:1788201706.629415:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.629417:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.629422:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.629430:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.629431:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.629433:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.629438:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.629440:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.629442:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.629444:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629452:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.629457:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.629458:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.629460:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.629462:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116671, 116671] 00000080:00200000:1.0:1788201706.629468:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.629470:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.629488:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.629490:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.629500:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477888512, count: 4096 00000080:00200000:1.0:1788201706.629506:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477888512, bytes: 4096 00000080:00200000:1.0:1788201706.629507:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.629509:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.629510:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477888512, 477892608) 0 0 00000080:00200000:1.0:1788201706.629512:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.629515:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116672, 116672] 00020000:00200000:1.0:1788201706.629517:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477888512 chunk: [477888512, 477892608) 477892608, 4096 00020000:00200000:1.0:1788201706.629518:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.629524:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116672, 116672] 00000008:00010000:1.0:1788201706.629527:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.629529:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.629533:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629539:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477888512 477892607) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629548:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629552:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629556:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.629560:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477888512, 477892608) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.629563:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116672 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.629567:0:17463:0:(rw26.c:898:ll_write_end()) pos 477888512, len 4096, copied 4096 00000080:00200000:1.0:1788201706.629569:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f343df1f[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116672] 00000080:00200000:1.0:1788201706.629570:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f959de84 fffffc0001029 2:0 ffff93c0c121f8c0 116672 lru 00000080:00200000:1.0:1788201706.629574:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ddb12650 116672: 1< 2 - - > 2< 477888512 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68257 - -> 00000080:00200000:1.0:1788201706.629575:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f343df1f 00000080:00200000:1.0:1788201706.629576:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.629578:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.629586:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.629598:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.629601:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.629604:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.629612:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.629616:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.629619:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.629622:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629630:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.629638:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.629641:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.629644:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.629648:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116672, 116672] 00000080:00200000:1.0:1788201706.629658:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.629662:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.629682:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.629684:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.629695:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477892608, count: 4096 00000080:00200000:1.0:1788201706.629701:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477892608, bytes: 4096 00000080:00200000:1.0:1788201706.629703:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.629705:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.629706:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477892608, 477896704) 0 0 00000080:00200000:1.0:1788201706.629707:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.629711:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116673, 116673] 00020000:00200000:1.0:1788201706.629713:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477892608 chunk: [477892608, 477896704) 477896704, 4096 00020000:00200000:1.0:1788201706.629714:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.629723:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116673, 116673] 00000008:00010000:1.0:1788201706.629728:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.629730:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.629737:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629751:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477892608 477896703) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629758:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629764:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629770:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.629777:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477892608, 477896704) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.629782:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116673 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.629787:0:17463:0:(rw26.c:898:ll_write_end()) pos 477892608, len 4096, copied 4096 00000080:00200000:1.0:1788201706.629790:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b85146eb[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116673] 00000080:00200000:1.0:1788201706.629793:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c9637424 fffffc0001029 2:0 ffff93c0c121fee0 116673 lru 00000080:00200000:1.0:1788201706.629798:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c788ef40 116673: 1< 2 - - > 2< 477892608 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68258 - -> 00000080:00200000:1.0:1788201706.629800:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b85146eb 00000080:00200000:1.0:1788201706.629801:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.629802:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.629810:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.629821:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.629824:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.629826:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.629852:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.629855:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.629857:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.629861:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.629868:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.629876:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.629878:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.629882:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.629886:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116673, 116673] 00000080:00200000:1.0:1788201706.629898:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.629900:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.629927:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.629930:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.629944:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477896704, count: 4096 00000080:00200000:1.0:1788201706.629953:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477896704, bytes: 4096 00000080:00200000:1.0:1788201706.629955:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.629959:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.629960:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477896704, 477900800) 0 0 00000080:00200000:1.0:1788201706.629962:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.629967:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116674, 116674] 00020000:00200000:1.0:1788201706.629970:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477896704 chunk: [477896704, 477900800) 477900800, 4096 00020000:00200000:1.0:1788201706.629972:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.629982:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116674, 116674] 00000008:00010000:1.0:1788201706.629987:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.629989:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.629997:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630005:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477896704 477900799) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630011:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630027:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630032:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.630037:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477896704, 477900800) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.630041:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116674 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.630045:0:17463:0:(rw26.c:898:ll_write_end()) pos 477896704, len 4096, copied 4096 00000080:00200000:1.0:1788201706.630048:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f499a93c[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116674] 00000080:00200000:1.0:1788201706.630049:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000da6ff24c fffffc0001029 2:0 ffff93c0c121f7e0 116674 lru 00000080:00200000:1.0:1788201706.630053:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000078866327 116674: 1< 2 - - > 2< 477896704 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68259 - -> 00000080:00200000:1.0:1788201706.630054:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f499a93c 00000080:00200000:1.0:1788201706.630055:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.630057:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.630063:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.630070:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.630073:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.630074:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.630080:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.630082:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.630084:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.630086:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630091:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.630096:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.630097:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.630100:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.630102:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116674, 116674] 00000080:00200000:1.0:1788201706.630109:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.630111:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.630129:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.630131:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.630141:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477900800, count: 4096 00000080:00200000:1.0:1788201706.630147:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477900800, bytes: 4096 00000080:00200000:1.0:1788201706.630149:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.630151:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.630152:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477900800, 477904896) 0 0 00000080:00200000:1.0:1788201706.630154:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.630157:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116675, 116675] 00020000:00200000:1.0:1788201706.630159:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477900800 chunk: [477900800, 477904896) 477904896, 4096 00020000:00200000:1.0:1788201706.630160:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.630167:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116675, 116675] 00000008:00010000:1.0:1788201706.630170:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.630171:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.630176:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630182:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477900800 477904895) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630188:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630192:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630196:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.630202:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477900800, 477904896) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.630206:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116675 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.630211:0:17463:0:(rw26.c:898:ll_write_end()) pos 477900800, len 4096, copied 4096 00000080:00200000:1.0:1788201706.630215:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007cd819d5[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116675] 00000080:00200000:1.0:1788201706.630218:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006df6796c fffffc0001029 2:0 ffff93c0c121f2a0 116675 lru 00000080:00200000:1.0:1788201706.630223:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000afaea7f9 116675: 1< 2 - - > 2< 477900800 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68260 - -> 00000080:00200000:1.0:1788201706.630226:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007cd819d5 00000080:00200000:1.0:1788201706.630227:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.630229:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.630237:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.630252:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.630254:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.630255:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.630262:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.630264:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.630265:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.630268:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630273:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.630278:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.630280:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.630282:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.630284:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116675, 116675] 00000080:00200000:1.0:1788201706.630291:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.630293:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.630311:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.630313:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.630323:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477904896, count: 4096 00000080:00200000:1.0:1788201706.630329:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477904896, bytes: 4096 00000080:00200000:1.0:1788201706.630331:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.630333:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.630335:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477904896, 477908992) 0 0 00000080:00200000:1.0:1788201706.630336:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.630339:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116676, 116676] 00020000:00200000:1.0:1788201706.630341:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477904896 chunk: [477904896, 477908992) 477908992, 4096 00020000:00200000:1.0:1788201706.630343:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.630349:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116676, 116676] 00000008:00010000:1.0:1788201706.630352:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.630354:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.630359:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630364:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477904896 477908991) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630369:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630374:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630378:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.630383:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477904896, 477908992) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.630386:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116676 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.630390:0:17463:0:(rw26.c:898:ll_write_end()) pos 477904896, len 4096, copied 4096 00000080:00200000:1.0:1788201706.630392:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009376a462[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116676] 00000080:00200000:1.0:1788201706.630394:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b4d7ff96 fffffc0001029 2:0 ffff93c0c121fe00 116676 lru 00000080:00200000:1.0:1788201706.630397:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b0cbee06 116676: 1< 2 - - > 2< 477904896 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68261 - -> 00000080:00200000:1.0:1788201706.630399:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009376a462 00000080:00200000:1.0:1788201706.630399:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.630401:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.630407:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.630414:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.630416:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.630417:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.630423:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.630425:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.630426:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.630428:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630433:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.630437:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.630439:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.630441:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.630443:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116676, 116676] 00000080:00200000:1.0:1788201706.630450:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.630452:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.630469:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.630471:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.630481:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477908992, count: 4096 00000080:00200000:1.0:1788201706.630487:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477908992, bytes: 4096 00000080:00200000:1.0:1788201706.630489:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.630491:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.630492:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477908992, 477913088) 0 0 00000080:00200000:1.0:1788201706.630494:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.630497:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116677, 116677] 00020000:00200000:1.0:1788201706.630499:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477908992 chunk: [477908992, 477913088) 477913088, 4096 00020000:00200000:1.0:1788201706.630500:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.630507:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116677, 116677] 00000008:00010000:1.0:1788201706.630510:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.630511:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.630516:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630522:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477908992 477913087) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630527:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630532:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630536:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.630540:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477908992, 477913088) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.630543:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116677 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.630546:0:17463:0:(rw26.c:898:ll_write_end()) pos 477908992, len 4096, copied 4096 00000080:00200000:1.0:1788201706.630549:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000035fdafe3[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116677] 00000080:00200000:1.0:1788201706.630551:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007d08efc4 fffffc0001029 2:0 ffff93c0c121f1c0 116677 lru 00000080:00200000:1.0:1788201706.630554:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007ed0c588 116677: 1< 2 - - > 2< 477908992 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68262 - -> 00000080:00200000:1.0:1788201706.630557:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000035fdafe3 00000080:00200000:1.0:1788201706.630558:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.630559:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.630565:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.630572:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.630574:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.630575:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.630581:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.630583:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.630584:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.630587:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630591:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.630596:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.630597:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.630599:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.630601:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116677, 116677] 00000080:00200000:1.0:1788201706.630608:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.630609:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.630627:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.630629:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.630639:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477913088, count: 4096 00000080:00200000:1.0:1788201706.630644:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477913088, bytes: 4096 00000080:00200000:1.0:1788201706.630647:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.630649:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.630650:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477913088, 477917184) 0 0 00000080:00200000:1.0:1788201706.630651:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.630654:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116678, 116678] 00020000:00200000:1.0:1788201706.630656:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477913088 chunk: [477913088, 477917184) 477917184, 4096 00020000:00200000:1.0:1788201706.630657:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.630664:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116678, 116678] 00000008:00010000:1.0:1788201706.630667:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.630668:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.630673:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630678:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477913088 477917183) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630684:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630688:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630692:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.630696:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477913088, 477917184) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.630699:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116678 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.630702:0:17463:0:(rw26.c:898:ll_write_end()) pos 477913088, len 4096, copied 4096 00000080:00200000:1.0:1788201706.630704:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d2a97de3[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116678] 00000080:00200000:1.0:1788201706.630707:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000000d3ca5 fffffc0001029 2:0 ffff93c0c121f620 116678 lru 00000080:00200000:1.0:1788201706.630710:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b7e6b981 116678: 1< 2 - - > 2< 477913088 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68263 - -> 00000080:00200000:1.0:1788201706.630712:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d2a97de3 00000080:00200000:1.0:1788201706.630713:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.630714:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.630719:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.630726:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.630728:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.630729:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.630735:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.630737:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.630738:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.630741:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630745:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.630751:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.630752:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.630755:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.630757:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116678, 116678] 00000080:00200000:1.0:1788201706.630763:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.630765:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.630782:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.630785:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.630795:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477917184, count: 4096 00000080:00200000:1.0:1788201706.630801:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477917184, bytes: 4096 00000080:00200000:1.0:1788201706.630803:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.630805:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.630806:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477917184, 477921280) 0 0 00000080:00200000:1.0:1788201706.630807:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.630810:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116679, 116679] 00020000:00200000:1.0:1788201706.630812:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477917184 chunk: [477917184, 477921280) 477921280, 4096 00020000:00200000:1.0:1788201706.630814:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.630820:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116679, 116679] 00000008:00010000:1.0:1788201706.630823:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.630824:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.630841:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630848:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477917184 477921279) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630853:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630858:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630862:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.630867:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477917184, 477921280) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.630870:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116679 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.630874:0:17463:0:(rw26.c:898:ll_write_end()) pos 477917184, len 4096, copied 4096 00000080:00200000:1.0:1788201706.630876:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008ba48860[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116679] 00000080:00200000:1.0:1788201706.630878:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000d99ce07 fffffc0001029 2:0 ffff93c0c121fd20 116679 lru 00000080:00200000:1.0:1788201706.630881:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000087647a95 116679: 1< 2 - - > 2< 477917184 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68264 - -> 00000080:00200000:1.0:1788201706.630883:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008ba48860 00000080:00200000:1.0:1788201706.630883:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.630885:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.630890:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.630898:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.630900:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.630901:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.630907:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.630908:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.630910:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.630912:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.630917:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.630922:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.630923:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.630925:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.630927:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116679, 116679] 00000080:00200000:1.0:1788201706.630934:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.630936:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.630954:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.630956:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.630966:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477921280, count: 4096 00000080:00200000:1.0:1788201706.630972:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477921280, bytes: 4096 00000080:00200000:1.0:1788201706.630974:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.630976:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.630977:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477921280, 477925376) 0 0 00000080:00200000:1.0:1788201706.630978:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.630981:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116680, 116680] 00020000:00200000:1.0:1788201706.630983:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477921280 chunk: [477921280, 477925376) 477925376, 4096 00020000:00200000:1.0:1788201706.630985:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.630991:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116680, 116680] 00000008:00010000:1.0:1788201706.630994:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.630995:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.631000:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631006:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477921280 477925375) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631011:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631028:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631033:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.631038:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477921280, 477925376) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.631042:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116680 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.631046:0:17463:0:(rw26.c:898:ll_write_end()) pos 477921280, len 4096, copied 4096 00000080:00200000:1.0:1788201706.631048:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b33f8c37[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116680] 00000080:00200000:1.0:1788201706.631050:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000055240e88 fffffc0001029 2:0 ffff93c0c121f9a0 116680 lru 00000080:00200000:1.0:1788201706.631053:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009918d372 116680: 1< 2 - - > 2< 477921280 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68265 - -> 00000080:00200000:1.0:1788201706.631055:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b33f8c37 00000080:00200000:1.0:1788201706.631055:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.631057:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.631063:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.631070:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.631072:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.631073:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.631079:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.631081:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.631083:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.631085:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631090:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.631095:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.631096:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.631098:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.631100:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116680, 116680] 00000080:00200000:1.0:1788201706.631107:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.631109:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.631135:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.631140:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.631155:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477925376, count: 4096 00000080:00200000:1.0:1788201706.631164:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477925376, bytes: 4096 00000080:00200000:1.0:1788201706.631166:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.631169:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.631170:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477925376, 477929472) 0 0 00000080:00200000:1.0:1788201706.631173:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.631178:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116681, 116681] 00020000:00200000:1.0:1788201706.631181:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477925376 chunk: [477925376, 477929472) 477929472, 4096 00020000:00200000:1.0:1788201706.631183:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.631194:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116681, 116681] 00000008:00010000:1.0:1788201706.631199:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.631201:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.631208:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631217:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477925376 477929471) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631225:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631231:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631237:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.631249:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477925376, 477929472) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.631254:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116681 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.631258:0:17463:0:(rw26.c:898:ll_write_end()) pos 477925376, len 4096, copied 4096 00000080:00200000:1.0:1788201706.631262:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003d95641a[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116681] 00000080:00200000:1.0:1788201706.631264:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e9e278f3 fffffc0001029 2:0 ffff93c0c121fb60 116681 lru 00000080:00200000:1.0:1788201706.631269:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007a7ef31e 116681: 1< 2 - - > 2< 477925376 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68266 - -> 00000080:00200000:1.0:1788201706.631271:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003d95641a 00000080:00200000:1.0:1788201706.631271:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.631273:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.631281:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.631291:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.631294:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.631295:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.631304:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.631306:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.631308:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.631312:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631319:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.631326:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.631328:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.631333:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.631336:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116681, 116681] 00000080:00200000:1.0:1788201706.631347:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.631349:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.631369:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.631372:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.631382:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477929472, count: 4096 00000080:00200000:1.0:1788201706.631388:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477929472, bytes: 4096 00000080:00200000:1.0:1788201706.631390:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.631392:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.631393:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477929472, 477933568) 0 0 00000080:00200000:1.0:1788201706.631395:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.631398:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116682, 116682] 00020000:00200000:1.0:1788201706.631400:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477929472 chunk: [477929472, 477933568) 477933568, 4096 00020000:00200000:1.0:1788201706.631401:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.631408:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116682, 116682] 00000008:00010000:1.0:1788201706.631411:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.631412:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.631418:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631423:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477929472 477933567) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631429:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631433:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631441:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.631445:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477929472, 477933568) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.631448:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116682 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.631452:0:17463:0:(rw26.c:898:ll_write_end()) pos 477929472, len 4096, copied 4096 00000080:00200000:1.0:1788201706.631454:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004172783f[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116682] 00000080:00200000:1.0:1788201706.631456:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c4dcb5d2 fffffc0001029 2:0 ffff93c0c121fc40 116682 lru 00000080:00200000:1.0:1788201706.631459:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000056e9ba2e 116682: 1< 2 - - > 2< 477929472 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68267 - -> 00000080:00200000:1.0:1788201706.631461:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004172783f 00000080:00200000:1.0:1788201706.631461:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.631463:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.631468:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.631476:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.631477:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.631479:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.631485:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.631486:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.631488:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.631490:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631495:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.631503:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.631505:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.631507:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.631509:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116682, 116682] 00000080:00200000:1.0:1788201706.631516:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.631518:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.631535:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.631537:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.631547:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477933568, count: 4096 00000080:00200000:1.0:1788201706.631553:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477933568, bytes: 4096 00000080:00200000:1.0:1788201706.631555:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.631557:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.631558:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477933568, 477937664) 0 0 00000080:00200000:1.0:1788201706.631560:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.631563:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116683, 116683] 00020000:00200000:1.0:1788201706.631565:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477933568 chunk: [477933568, 477937664) 477937664, 4096 00020000:00200000:1.0:1788201706.631566:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.631572:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116683, 116683] 00000008:00010000:1.0:1788201706.631575:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.631577:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.631581:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631587:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477933568 477937663) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631592:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631601:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631605:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.631609:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477933568, 477937664) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.631612:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116683 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.631615:0:17463:0:(rw26.c:898:ll_write_end()) pos 477933568, len 4096, copied 4096 00000080:00200000:1.0:1788201706.631618:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b69a6757[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116683] 00000080:00200000:1.0:1788201706.631619:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007732cc3a fffffc0001029 2:0 ffff93c0c121f540 116683 lru 00000080:00200000:1.0:1788201706.631623:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000aed95109 116683: 1< 2 - - > 2< 477933568 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68268 - -> 00000080:00200000:1.0:1788201706.631624:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b69a6757 00000080:00200000:1.0:1788201706.631624:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.631626:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.631631:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.631639:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.631640:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.631642:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.631647:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.631649:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.631650:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.631653:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631661:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.631666:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.631667:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.631669:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.631671:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116683, 116683] 00000080:00200000:1.0:1788201706.631677:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.631679:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.631697:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.631699:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.631709:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477937664, count: 4096 00000080:00200000:1.0:1788201706.631715:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477937664, bytes: 4096 00000080:00200000:1.0:1788201706.631717:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.631719:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.631720:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477937664, 477941760) 0 0 00000080:00200000:1.0:1788201706.631722:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.631725:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116684, 116684] 00020000:00200000:1.0:1788201706.631726:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477937664 chunk: [477937664, 477941760) 477941760, 4096 00020000:00200000:1.0:1788201706.631728:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.631734:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116684, 116684] 00000008:00010000:1.0:1788201706.631737:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.631738:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.631745:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631754:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477937664 477941759) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631768:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631775:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631782:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.631790:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477937664, 477941760) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.631795:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116684 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.631800:0:17463:0:(rw26.c:898:ll_write_end()) pos 477937664, len 4096, copied 4096 00000080:00200000:1.0:1788201706.631803:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000080ca021f[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116684] 00000080:00200000:1.0:1788201706.631805:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000aafc2182 fffffc0001029 2:0 ffff93c0c121f460 116684 lru 00000080:00200000:1.0:1788201706.631809:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005d155cd2 116684: 1< 2 - - > 2< 477937664 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68269 - -> 00000080:00200000:1.0:1788201706.631810:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000080ca021f 00000080:00200000:1.0:1788201706.631811:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.631812:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.631818:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.631826:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.631828:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.631842:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.631849:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.631851:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.631853:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.631855:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631861:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.631866:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.631867:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.631870:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.631872:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116684, 116684] 00000080:00200000:1.0:1788201706.631879:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.631881:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.631899:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.631901:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.631911:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477941760, count: 4096 00000080:00200000:1.0:1788201706.631918:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477941760, bytes: 4096 00000080:00200000:1.0:1788201706.631919:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.631921:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.631922:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477941760, 477945856) 0 0 00000080:00200000:1.0:1788201706.631924:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.631927:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116685, 116685] 00020000:00200000:1.0:1788201706.631929:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477941760 chunk: [477941760, 477945856) 477945856, 4096 00020000:00200000:1.0:1788201706.631930:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.631937:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116685, 116685] 00000008:00010000:1.0:1788201706.631940:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.631941:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.631946:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631956:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477941760 477945855) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631961:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631965:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.631970:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.631974:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477941760, 477945856) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.631977:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116685 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.631980:0:17463:0:(rw26.c:898:ll_write_end()) pos 477941760, len 4096, copied 4096 00000080:00200000:1.0:1788201706.631982:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002cc70f72[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116685] 00000080:00200000:1.0:1788201706.631984:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000827d1f47 fffffc0001029 2:0 ffff93c0c121f0e0 116685 lru 00000080:00200000:1.0:1788201706.631987:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002fb79785 116685: 1< 2 - - > 2< 477941760 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68270 - -> 00000080:00200000:1.0:1788201706.631989:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002cc70f72 00000080:00200000:1.0:1788201706.631989:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.631991:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.631996:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.632003:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.632005:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.632007:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.632013:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.632026:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.632030:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.632033:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632038:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.632043:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.632044:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.632047:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.632049:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116685, 116685] 00000080:00200000:1.0:1788201706.632056:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.632059:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.632077:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.632079:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.632090:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477945856, count: 4096 00000080:00200000:1.0:1788201706.632096:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477945856, bytes: 4096 00000080:00200000:1.0:1788201706.632097:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.632099:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.632100:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477945856, 477949952) 0 0 00000080:00200000:1.0:1788201706.632102:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.632105:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116686, 116686] 00020000:00200000:1.0:1788201706.632107:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477945856 chunk: [477945856, 477949952) 477949952, 4096 00020000:00200000:1.0:1788201706.632109:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.632115:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116686, 116686] 00000008:00010000:1.0:1788201706.632118:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.632119:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.632125:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632131:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477945856 477949951) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632136:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632141:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632145:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.632149:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477945856, 477949952) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.632152:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116686 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.632156:0:17463:0:(rw26.c:898:ll_write_end()) pos 477945856, len 4096, copied 4096 00000080:00200000:1.0:1788201706.632158:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000023ee4f1a[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116686] 00000080:00200000:1.0:1788201706.632159:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000111f2e68 fffffc0001029 2:0 ffff93c0c121f700 116686 lru 00000080:00200000:1.0:1788201706.632163:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000038d1ae10 116686: 1< 2 - - > 2< 477945856 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68271 - -> 00000080:00200000:1.0:1788201706.632164:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000023ee4f1a 00000080:00200000:1.0:1788201706.632165:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.632166:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.632172:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.632179:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.632181:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.632183:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.632188:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.632190:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.632192:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.632194:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632199:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.632204:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.632205:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.632207:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.632209:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116686, 116686] 00000080:00200000:1.0:1788201706.632216:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.632218:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.632235:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.632237:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.632247:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477949952, count: 4096 00000080:00200000:1.0:1788201706.632253:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477949952, bytes: 4096 00000080:00200000:1.0:1788201706.632255:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.632257:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.632258:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477949952, 477954048) 0 0 00000080:00200000:1.0:1788201706.632260:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.632263:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116687, 116687] 00020000:00200000:1.0:1788201706.632265:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477949952 chunk: [477949952, 477954048) 477954048, 4096 00020000:00200000:1.0:1788201706.632266:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.632272:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116687, 116687] 00000008:00010000:1.0:1788201706.632276:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.632277:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.632282:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632287:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477949952 477954047) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632293:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632297:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632301:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.632306:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477949952, 477954048) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.632309:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116687 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.632312:0:17463:0:(rw26.c:898:ll_write_end()) pos 477949952, len 4096, copied 4096 00000080:00200000:1.0:1788201706.632314:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006bac6976[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116687] 00000080:00200000:1.0:1788201706.632316:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d419ab50 fffffc0001029 2:0 ffff93c0c121f000 116687 lru 00000080:00200000:1.0:1788201706.632319:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cfef9ae5 116687: 1< 2 - - > 2< 477949952 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68272 - -> 00000080:00200000:1.0:1788201706.632321:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006bac6976 00000080:00200000:1.0:1788201706.632322:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.632323:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.632328:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.632339:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.632340:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.632342:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.632347:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.632349:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.632351:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.632353:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632357:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.632363:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.632364:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.632367:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.632368:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116687, 116687] 00000080:00200000:1.0:1788201706.632375:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.632377:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.632394:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.632396:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.632406:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477954048, count: 4096 00000080:00200000:1.0:1788201706.632412:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477954048, bytes: 4096 00000080:00200000:1.0:1788201706.632414:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.632416:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.632418:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477954048, 477958144) 0 0 00000080:00200000:1.0:1788201706.632419:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.632422:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116688, 116688] 00020000:00200000:1.0:1788201706.632424:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477954048 chunk: [477954048, 477958144) 477958144, 4096 00020000:00200000:1.0:1788201706.632426:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.632432:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116688, 116688] 00000008:00010000:1.0:1788201706.632435:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.632436:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.632441:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632447:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477954048 477958143) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632452:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632456:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632460:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.632465:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477954048, 477958144) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.632468:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116688 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.632472:0:17463:0:(rw26.c:898:ll_write_end()) pos 477954048, len 4096, copied 4096 00000080:00200000:1.0:1788201706.632474:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f2db66c7[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116688] 00000080:00200000:1.0:1788201706.632476:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006a23e82b fffffc0001029 2:0 ffff93c0c12207e0 116688 lru 00000080:00200000:1.0:1788201706.632479:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004cb6657d 116688: 1< 2 - - > 2< 477954048 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68273 - -> 00000080:00200000:1.0:1788201706.632481:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f2db66c7 00000080:00200000:1.0:1788201706.632481:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.632483:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.632488:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.632496:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.632497:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.632499:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.632504:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.632506:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.632507:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.632510:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632514:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.632519:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.632520:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.632523:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.632524:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116688, 116688] 00000080:00200000:1.0:1788201706.632531:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.632533:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.632550:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.632552:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.632562:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477958144, count: 4096 00000080:00200000:1.0:1788201706.632568:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477958144, bytes: 4096 00000080:00200000:1.0:1788201706.632570:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.632572:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.632573:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477958144, 477962240) 0 0 00000080:00200000:1.0:1788201706.632575:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.632578:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116689, 116689] 00020000:00200000:1.0:1788201706.632580:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477958144 chunk: [477958144, 477962240) 477962240, 4096 00020000:00200000:1.0:1788201706.632581:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.632588:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116689, 116689] 00000008:00010000:1.0:1788201706.632590:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.632592:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.632597:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632602:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477958144 477962239) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632608:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632612:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632616:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.632621:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477958144, 477962240) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.632623:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116689 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.632627:0:17463:0:(rw26.c:898:ll_write_end()) pos 477958144, len 4096, copied 4096 00000080:00200000:1.0:1788201706.632629:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001d57ffcf[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116689] 00000080:00200000:1.0:1788201706.632631:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000852dc9a fffffc0001029 2:0 ffff93c0c12202a0 116689 lru 00000080:00200000:1.0:1788201706.632635:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000019cdbd22 116689: 1< 2 - - > 2< 477958144 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68274 - -> 00000080:00200000:1.0:1788201706.632637:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001d57ffcf 00000080:00200000:1.0:1788201706.632638:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.632639:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.632644:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.632652:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.632653:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.632655:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.632660:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.632662:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.632664:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.632666:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632670:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.632675:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.632677:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.632679:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.632681:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116689, 116689] 00000080:00200000:1.0:1788201706.632688:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.632689:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.632707:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.632709:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.632719:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477962240, count: 4096 00000080:00200000:1.0:1788201706.632724:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477962240, bytes: 4096 00000080:00200000:1.0:1788201706.632726:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.632729:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.632730:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477962240, 477966336) 0 0 00000080:00200000:1.0:1788201706.632731:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.632734:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116690, 116690] 00020000:00200000:1.0:1788201706.632737:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477962240 chunk: [477962240, 477966336) 477966336, 4096 00020000:00200000:1.0:1788201706.632739:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.632749:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116690, 116690] 00000008:00010000:1.0:1788201706.632754:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.632756:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.632763:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632772:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477962240 477966335) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632781:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632788:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632796:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.632804:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477962240, 477966336) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.632809:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116690 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.632815:0:17463:0:(rw26.c:898:ll_write_end()) pos 477962240, len 4096, copied 4096 00000080:00200000:1.0:1788201706.632818:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000037daea14[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116690] 00000080:00200000:1.0:1788201706.632822:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003988ee85 fffffc0001029 2:0 ffff93c0c1220e00 116690 lru 00000080:00200000:1.0:1788201706.632828:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000db3942f6 116690: 1< 2 - - > 2< 477962240 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68275 - -> 00000080:00200000:1.0:1788201706.632846:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000037daea14 00000080:00200000:1.0:1788201706.632846:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.632848:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.632855:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.632863:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.632865:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.632866:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.632872:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.632874:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.632876:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.632878:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632883:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.632888:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.632890:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.632892:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.632894:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116690, 116690] 00000080:00200000:1.0:1788201706.632901:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.632903:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.632921:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.632924:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.632934:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477966336, count: 4096 00000080:00200000:1.0:1788201706.632941:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477966336, bytes: 4096 00000080:00200000:1.0:1788201706.632942:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.632945:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.632945:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477966336, 477970432) 0 0 00000080:00200000:1.0:1788201706.632947:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.632950:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116691, 116691] 00020000:00200000:1.0:1788201706.632952:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477966336 chunk: [477966336, 477970432) 477970432, 4096 00020000:00200000:1.0:1788201706.632954:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.632960:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116691, 116691] 00000008:00010000:1.0:1788201706.632963:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.632965:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.632970:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632975:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477966336 477970431) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632981:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632986:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.632989:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.632993:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477966336, 477970432) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.632996:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116691 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.633000:0:17463:0:(rw26.c:898:ll_write_end()) pos 477966336, len 4096, copied 4096 00000080:00200000:1.0:1788201706.633002:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008f7f4fa5[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116691] 00000080:00200000:1.0:1788201706.633004:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002b35530b fffffc0001029 2:0 ffff93c0c12201c0 116691 lru 00000080:00200000:1.0:1788201706.633008:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009a8581a3 116691: 1< 2 - - > 2< 477966336 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68276 - -> 00000080:00200000:1.0:1788201706.633009:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008f7f4fa5 00000080:00200000:1.0:1788201706.633010:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.633011:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.633029:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.633037:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.633039:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.633041:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.633047:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.633049:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.633051:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.633053:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633058:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.633063:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.633064:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.633067:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.633069:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116691, 116691] 00000080:00200000:1.0:1788201706.633076:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.633078:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.633096:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.633099:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.633109:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477970432, count: 4096 00000080:00200000:1.0:1788201706.633115:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477970432, bytes: 4096 00000080:00200000:1.0:1788201706.633116:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.633119:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.633120:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477970432, 477974528) 0 0 00000080:00200000:1.0:1788201706.633121:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.633124:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116692, 116692] 00020000:00200000:1.0:1788201706.633126:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477970432 chunk: [477970432, 477974528) 477974528, 4096 00020000:00200000:1.0:1788201706.633128:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.633134:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116692, 116692] 00000008:00010000:1.0:1788201706.633137:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.633138:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.633143:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633149:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477970432 477974527) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633157:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633165:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633172:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.633180:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477970432, 477974528) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.633186:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116692 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.633191:0:17463:0:(rw26.c:898:ll_write_end()) pos 477970432, len 4096, copied 4096 00000080:00200000:1.0:1788201706.633195:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000006442ad0[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116692] 00000080:00200000:1.0:1788201706.633198:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000930f9aa1 fffffc0001029 2:0 ffff93c0c1220620 116692 lru 00000080:00200000:1.0:1788201706.633203:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000033a015e7 116692: 1< 2 - - > 2< 477970432 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68277 - -> 00000080:00200000:1.0:1788201706.633205:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000006442ad0 00000080:00200000:1.0:1788201706.633206:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.633208:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.633217:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.633229:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.633231:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.633233:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.633242:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.633245:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.633247:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.633252:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633260:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.633267:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.633270:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.633274:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.633277:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116692, 116692] 00000080:00200000:1.0:1788201706.633288:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.633291:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.633319:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.633322:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.633338:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477974528, count: 4096 00000080:00200000:1.0:1788201706.633348:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477974528, bytes: 4096 00000080:00200000:1.0:1788201706.633351:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.633355:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.633357:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477974528, 477978624) 0 0 00000080:00200000:1.0:1788201706.633359:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.633364:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116693, 116693] 00020000:00200000:1.0:1788201706.633367:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477974528 chunk: [477974528, 477978624) 477978624, 4096 00020000:00200000:1.0:1788201706.633369:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.633381:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116693, 116693] 00000008:00010000:1.0:1788201706.633387:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.633389:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.633397:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633405:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477974528 477978623) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633414:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633421:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633428:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.633440:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477974528, 477978624) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.633445:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116693 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.633451:0:17463:0:(rw26.c:898:ll_write_end()) pos 477974528, len 4096, copied 4096 00000080:00200000:1.0:1788201706.633456:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000016a30f0[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116693] 00000080:00200000:1.0:1788201706.633459:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004879c30f fffffc0001029 2:0 ffff93c0c1220d20 116693 lru 00000080:00200000:1.0:1788201706.633465:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000087aa6804 116693: 1< 2 - - > 2< 477974528 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68278 - -> 00000080:00200000:1.0:1788201706.633466:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000016a30f0 00000080:00200000:1.0:1788201706.633468:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.633470:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.633478:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.633489:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.633492:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.633494:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.633503:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.633506:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.633508:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.633512:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633520:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.633528:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.633531:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.633535:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.633538:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116693, 116693] 00000080:00200000:1.0:1788201706.633548:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.633552:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.633577:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.633581:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.633596:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477978624, count: 4096 00000080:00200000:1.0:1788201706.633606:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477978624, bytes: 4096 00000080:00200000:1.0:1788201706.633609:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.633613:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.633614:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477978624, 477982720) 0 0 00000080:00200000:1.0:1788201706.633617:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.633621:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116694, 116694] 00020000:00200000:1.0:1788201706.633624:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477978624 chunk: [477978624, 477982720) 477982720, 4096 00020000:00200000:1.0:1788201706.633627:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.633638:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116694, 116694] 00000008:00010000:1.0:1788201706.633643:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.633645:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.633652:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633660:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477978624 477982719) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633667:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633674:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633686:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.633694:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477978624, 477982720) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.633699:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116694 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.633704:0:17463:0:(rw26.c:898:ll_write_end()) pos 477978624, len 4096, copied 4096 00000080:00200000:1.0:1788201706.633708:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002e2e178d[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116694] 00000080:00200000:1.0:1788201706.633710:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000077ec770e fffffc0001029 2:0 ffff93c0c12209a0 116694 lru 00000080:00200000:1.0:1788201706.633716:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004b6e0f1f 116694: 1< 2 - - > 2< 477978624 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68279 - -> 00000080:00200000:1.0:1788201706.633718:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002e2e178d 00000080:00200000:1.0:1788201706.633719:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.633721:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.633730:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.633742:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.633744:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.633747:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.633756:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.633759:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.633761:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.633765:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633772:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.633785:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.633787:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.633791:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.633794:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116694, 116694] 00000080:00200000:1.0:1788201706.633805:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.633809:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.633866:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.633870:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.633887:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477982720, count: 4096 00000080:00200000:1.0:1788201706.633898:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477982720, bytes: 4096 00000080:00200000:1.0:1788201706.633901:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.633905:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.633907:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477982720, 477986816) 0 0 00000080:00200000:1.0:1788201706.633909:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.633914:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116695, 116695] 00020000:00200000:1.0:1788201706.633917:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477982720 chunk: [477982720, 477986816) 477986816, 4096 00020000:00200000:1.0:1788201706.633919:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.633932:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116695, 116695] 00000008:00010000:1.0:1788201706.633937:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.633939:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.633947:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633956:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477982720 477986815) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633965:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633978:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.633986:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.633993:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477982720, 477986816) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.633998:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116695 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.634003:0:17463:0:(rw26.c:898:ll_write_end()) pos 477982720, len 4096, copied 4096 00000080:00200000:1.0:1788201706.634008:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000067ba5aee[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116695] 00000080:00200000:1.0:1788201706.634011:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000ce12496 fffffc0001029 2:0 ffff93c0c1220b60 116695 lru 00000080:00200000:1.0:1788201706.634030:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009f49f6c5 116695: 1< 2 - - > 2< 477982720 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68280 - -> 00000080:00200000:1.0:1788201706.634032:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000067ba5aee 00000080:00200000:1.0:1788201706.634034:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.634036:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.634045:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.634057:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.634060:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.634062:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.634072:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.634075:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.634077:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.634081:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634094:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.634103:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.634106:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.634110:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.634113:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116695, 116695] 00000080:00200000:1.0:1788201706.634125:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.634128:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.634159:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.634162:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.634179:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477986816, count: 4096 00000080:00200000:1.0:1788201706.634190:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477986816, bytes: 4096 00000080:00200000:1.0:1788201706.634193:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.634197:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.634198:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477986816, 477990912) 0 0 00000080:00200000:1.0:1788201706.634201:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.634206:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116696, 116696] 00020000:00200000:1.0:1788201706.634209:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477986816 chunk: [477986816, 477990912) 477990912, 4096 00020000:00200000:1.0:1788201706.634211:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.634223:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116696, 116696] 00000008:00010000:1.0:1788201706.634229:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.634231:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.634238:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634248:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477986816 477990911) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634262:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634270:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634277:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.634285:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477986816, 477990912) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.634291:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116696 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.634296:0:17463:0:(rw26.c:898:ll_write_end()) pos 477986816, len 4096, copied 4096 00000080:00200000:1.0:1788201706.634300:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006444a443[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116696] 00000080:00200000:1.0:1788201706.634304:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000110d4151 fffffc0001029 2:0 ffff93c0c1220c40 116696 lru 00000080:00200000:1.0:1788201706.634310:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005dce17dd 116696: 1< 2 - - > 2< 477986816 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68281 - -> 00000080:00200000:1.0:1788201706.634312:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006444a443 00000080:00200000:1.0:1788201706.634313:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.634315:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.634324:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.634336:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.634339:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.634341:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.634351:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.634354:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.634357:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.634361:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634369:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.634378:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.634380:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.634385:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.634388:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116696, 116696] 00000080:00200000:1.0:1788201706.634400:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.634403:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.634432:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.634435:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.634452:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477990912, count: 4096 00000080:00200000:1.0:1788201706.634464:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477990912, bytes: 4096 00000080:00200000:1.0:1788201706.634467:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.634470:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.634472:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477990912, 477995008) 0 0 00000080:00200000:1.0:1788201706.634475:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.634480:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116697, 116697] 00020000:00200000:1.0:1788201706.634483:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477990912 chunk: [477990912, 477995008) 477995008, 4096 00020000:00200000:1.0:1788201706.634485:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.634497:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116697, 116697] 00000008:00010000:1.0:1788201706.634502:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.634504:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.634511:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634525:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477990912 477995007) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634533:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634541:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634548:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.634556:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477990912, 477995008) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.634561:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116697 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.634567:0:17463:0:(rw26.c:898:ll_write_end()) pos 477990912, len 4096, copied 4096 00000080:00200000:1.0:1788201706.634570:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bc67a181[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116697] 00000080:00200000:1.0:1788201706.634573:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003f89bf3f fffffc0001029 2:0 ffff93c0c1220540 116697 lru 00000080:00200000:1.0:1788201706.634579:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000026cea6f1 116697: 1< 2 - - > 2< 477990912 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68282 - -> 00000080:00200000:1.0:1788201706.634582:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bc67a181 00000080:00200000:1.0:1788201706.634583:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.634585:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.634593:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.634604:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.634607:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.634610:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.634619:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.634622:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.634624:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.634628:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634636:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.634644:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.634646:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.634650:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.634653:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116697, 116697] 00000080:00200000:1.0:1788201706.634663:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.634666:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.634695:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.634699:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.634714:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477995008, count: 4096 00000080:00200000:1.0:1788201706.634725:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477995008, bytes: 4096 00000080:00200000:1.0:1788201706.634728:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.634732:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.634733:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477995008, 477999104) 0 0 00000080:00200000:1.0:1788201706.634735:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.634740:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116698, 116698] 00020000:00200000:1.0:1788201706.634743:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477995008 chunk: [477995008, 477999104) 477999104, 4096 00020000:00200000:1.0:1788201706.634746:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.634757:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116698, 116698] 00000008:00010000:1.0:1788201706.634763:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.634765:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.634773:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634782:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477995008 477999103) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634791:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634799:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634806:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.634814:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477995008, 477999104) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.634820:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116698 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.634824:0:17463:0:(rw26.c:898:ll_write_end()) pos 477995008, len 4096, copied 4096 00000080:00200000:1.0:1788201706.634846:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fc2ce93a[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116698] 00000080:00200000:1.0:1788201706.634850:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003bd22a3c fffffc0001029 2:0 ffff93c0c1220460 116698 lru 00000080:00200000:1.0:1788201706.634856:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000061def3fc 116698: 1< 2 - - > 2< 477995008 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68283 - -> 00000080:00200000:1.0:1788201706.634859:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fc2ce93a 00000080:00200000:1.0:1788201706.634860:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.634862:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.634872:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.634884:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.634888:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.634890:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.634899:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.634902:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.634904:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.634909:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.634916:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.634925:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.634928:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.634933:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.634936:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116698, 116698] 00000080:00200000:1.0:1788201706.634947:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.634950:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.634976:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.634979:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.634994:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 477999104, count: 4096 00000080:00200000:1.0:1788201706.635006:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 477999104, bytes: 4096 00000080:00200000:1.0:1788201706.635008:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.635012:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.635014:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [477999104, 478003200) 0 0 00000080:00200000:1.0:1788201706.635029:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.635034:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116699, 116699] 00020000:00200000:1.0:1788201706.635038:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 477999104 chunk: [477999104, 478003200) 478003200, 4096 00020000:00200000:1.0:1788201706.635040:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.635053:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116699, 116699] 00000008:00010000:1.0:1788201706.635058:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.635061:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.635069:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635078:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (477999104 478003199) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635086:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635093:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635101:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.635109:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [477999104, 478003200) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.635114:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116699 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.635119:0:17463:0:(rw26.c:898:ll_write_end()) pos 477999104, len 4096, copied 4096 00000080:00200000:1.0:1788201706.635123:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005eb0260b[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116699] 00000080:00200000:1.0:1788201706.635126:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000aee38cac fffffc0001029 2:0 ffff93c0c12200e0 116699 lru 00000080:00200000:1.0:1788201706.635132:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000218f16e9 116699: 1< 2 - - > 2< 477999104 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68284 - -> 00000080:00200000:1.0:1788201706.635134:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005eb0260b 00000080:00200000:1.0:1788201706.635135:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.635137:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.635146:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.635165:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.635168:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.635171:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.635181:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.635185:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.635187:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.635191:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635200:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.635208:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.635211:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.635215:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.635218:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116699, 116699] 00000080:00200000:1.0:1788201706.635229:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.635232:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.635258:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.635262:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.635278:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478003200, count: 4096 00000080:00200000:1.0:1788201706.635288:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478003200, bytes: 4096 00000080:00200000:1.0:1788201706.635291:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.635294:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.635296:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478003200, 478007296) 0 0 00000080:00200000:1.0:1788201706.635298:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.635303:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116700, 116700] 00020000:00200000:1.0:1788201706.635307:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478003200 chunk: [478003200, 478007296) 478007296, 4096 00020000:00200000:1.0:1788201706.635309:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.635320:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116700, 116700] 00000008:00010000:1.0:1788201706.635326:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.635328:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.635336:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635344:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478003200 478007295) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635353:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635361:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635369:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.635377:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478003200, 478007296) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.635382:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116700 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.635387:0:17463:0:(rw26.c:898:ll_write_end()) pos 478003200, len 4096, copied 4096 00000080:00200000:1.0:1788201706.635391:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000019129e8d[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116700] 00000080:00200000:1.0:1788201706.635394:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bfd9d465 fffffc0001029 2:0 ffff93c0c1220700 116700 lru 00000080:00200000:1.0:1788201706.635399:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000eb598ef0 116700: 1< 2 - - > 2< 478003200 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68285 - -> 00000080:00200000:1.0:1788201706.635401:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000019129e8d 00000080:00200000:1.0:1788201706.635402:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.635404:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.635414:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.635426:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.635429:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.635431:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.635441:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.635444:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.635447:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.635450:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635458:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.635466:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.635469:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.635473:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.635477:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116700, 116700] 00000080:00200000:1.0:1788201706.635487:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.635490:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.635519:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.635522:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.635537:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478007296, count: 4096 00000080:00200000:1.0:1788201706.635547:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478007296, bytes: 4096 00000080:00200000:1.0:1788201706.635550:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.635553:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.635555:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478007296, 478011392) 0 0 00000080:00200000:1.0:1788201706.635558:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.635562:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116701, 116701] 00020000:00200000:1.0:1788201706.635565:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478007296 chunk: [478007296, 478011392) 478011392, 4096 00020000:00200000:1.0:1788201706.635567:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.635578:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116701, 116701] 00000008:00010000:1.0:1788201706.635582:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.635584:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.635592:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635601:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478007296 478011391) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635610:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635617:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635624:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.635632:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478007296, 478011392) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.635638:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116701 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.635643:0:17463:0:(rw26.c:898:ll_write_end()) pos 478007296, len 4096, copied 4096 00000080:00200000:1.0:1788201706.635647:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009e0b7b4d[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116701] 00000080:00200000:1.0:1788201706.635650:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006b1dd479 fffffc0001029 2:0 ffff93c0c1220000 116701 lru 00000080:00200000:1.0:1788201706.635656:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005e2fbed6 116701: 1< 2 - - > 2< 478007296 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68286 - -> 00000080:00200000:1.0:1788201706.635660:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009e0b7b4d 00000080:00200000:1.0:1788201706.635661:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.635663:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.635671:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.635682:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.635684:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.635686:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.635694:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.635697:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.635699:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.635703:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635709:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.635717:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.635719:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.635723:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.635726:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116701, 116701] 00000080:00200000:1.0:1788201706.635735:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.635738:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.635766:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.635770:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.635785:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478011392, count: 4096 00000080:00200000:1.0:1788201706.635795:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478011392, bytes: 4096 00000080:00200000:1.0:1788201706.635798:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.635802:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.635804:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478011392, 478015488) 0 0 00000080:00200000:1.0:1788201706.635806:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.635811:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116702, 116702] 00020000:00200000:1.0:1788201706.635814:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478011392 chunk: [478011392, 478015488) 478015488, 4096 00020000:00200000:1.0:1788201706.635816:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.635827:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116702, 116702] 00000008:00010000:1.0:1788201706.635850:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.635853:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.635860:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635869:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478011392 478015487) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635877:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635881:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635886:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.635890:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478011392, 478015488) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.635894:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116702 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.635898:0:17463:0:(rw26.c:898:ll_write_end()) pos 478011392, len 4096, copied 4096 00000080:00200000:1.0:1788201706.635900:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000181a0b5f[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116702] 00000080:00200000:1.0:1788201706.635903:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000088894275 fffffc0001029 2:0 ffff93c0c1220a80 116702 lru 00000080:00200000:1.0:1788201706.635906:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c217bbd2 116702: 1< 2 - - > 2< 478011392 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68287 - -> 00000080:00200000:1.0:1788201706.635908:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000181a0b5f 00000080:00200000:1.0:1788201706.635909:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.635910:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.635916:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.635924:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.635926:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.635927:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.635934:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.635935:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.635937:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.635939:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.635944:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.635949:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.635950:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.635953:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.635955:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116702, 116702] 00000080:00200000:1.0:1788201706.635962:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.635964:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.635982:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.635984:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.635995:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478015488, count: 4096 00000080:00200000:1.0:1788201706.636001:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478015488, bytes: 4096 00000080:00200000:1.0:1788201706.636003:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.636005:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.636006:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478015488, 478019584) 0 0 00000080:00200000:1.0:1788201706.636008:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.636011:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116703, 116703] 00020000:00200000:1.0:1788201706.636013:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478015488 chunk: [478015488, 478019584) 478019584, 4096 00020000:00200000:1.0:1788201706.636026:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.636037:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116703, 116703] 00000008:00010000:1.0:1788201706.636040:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.636042:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.636047:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636053:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478015488 478019583) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636058:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636063:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636067:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.636072:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478015488, 478019584) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.636075:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116703 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.636079:0:17463:0:(rw26.c:898:ll_write_end()) pos 478015488, len 4096, copied 4096 00000080:00200000:1.0:1788201706.636081:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004e8f4999[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116703] 00000080:00200000:1.0:1788201706.636083:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f258a994 fffffc0001029 2:0 ffff93c0c1220380 116703 lru 00000080:00200000:1.0:1788201706.636087:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d55e05ee 116703: 1< 2 - - > 2< 478015488 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68288 - -> 00000080:00200000:1.0:1788201706.636088:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004e8f4999 00000080:00200000:1.0:1788201706.636089:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.636090:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.636096:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.636104:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.636106:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.636107:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.636113:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.636115:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.636117:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.636119:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636124:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.636129:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.636130:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.636133:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.636135:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116703, 116703] 00000080:00200000:1.0:1788201706.636142:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.636144:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.636162:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.636164:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.636174:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478019584, count: 4096 00000080:00200000:1.0:1788201706.636180:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478019584, bytes: 4096 00000080:00200000:1.0:1788201706.636182:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.636184:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.636185:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478019584, 478023680) 0 0 00000080:00200000:1.0:1788201706.636187:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.636190:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116704, 116704] 00020000:00200000:1.0:1788201706.636192:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478019584 chunk: [478019584, 478023680) 478023680, 4096 00020000:00200000:1.0:1788201706.636194:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.636200:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116704, 116704] 00000008:00010000:1.0:1788201706.636203:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.636204:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.636209:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636214:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478019584 478023679) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636220:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636224:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636228:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.636233:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478019584, 478023680) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.636236:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116704 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.636240:0:17463:0:(rw26.c:898:ll_write_end()) pos 478019584, len 4096, copied 4096 00000080:00200000:1.0:1788201706.636242:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000041b10e94[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116704] 00000080:00200000:1.0:1788201706.636244:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000a442fc7 fffffc0001029 2:0 ffff93c0c12208c0 116704 lru 00000080:00200000:1.0:1788201706.636247:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000fc3f5cb0 116704: 1< 2 - - > 2< 478019584 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68289 - -> 00000080:00200000:1.0:1788201706.636249:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000041b10e94 00000080:00200000:1.0:1788201706.636249:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.636251:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.636256:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.636264:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.636266:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.636267:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.636273:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.636274:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.636276:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.636278:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636283:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.636288:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.636289:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.636291:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.636293:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116704, 116704] 00000080:00200000:1.0:1788201706.636300:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.636302:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.636320:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.636322:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.636332:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478023680, count: 4096 00000080:00200000:1.0:1788201706.636337:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478023680, bytes: 4096 00000080:00200000:1.0:1788201706.636339:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.636341:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.636342:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478023680, 478027776) 0 0 00000080:00200000:1.0:1788201706.636344:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.636347:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116705, 116705] 00020000:00200000:1.0:1788201706.636349:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478023680 chunk: [478023680, 478027776) 478027776, 4096 00020000:00200000:1.0:1788201706.636350:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.636356:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116705, 116705] 00000008:00010000:1.0:1788201706.636359:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.636361:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.636365:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636371:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478023680 478027775) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636376:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636380:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636384:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.636392:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478023680, 478027776) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.636395:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116705 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.636399:0:17463:0:(rw26.c:898:ll_write_end()) pos 478023680, len 4096, copied 4096 00000080:00200000:1.0:1788201706.636401:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000cc1a2353[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116705] 00000080:00200000:1.0:1788201706.636403:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000086cc2cf8 fffffc0001029 2:0 ffff93c0c1220ee0 116705 lru 00000080:00200000:1.0:1788201706.636406:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006c5e3dc7 116705: 1< 2 - - > 2< 478023680 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68290 - -> 00000080:00200000:1.0:1788201706.636408:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000cc1a2353 00000080:00200000:1.0:1788201706.636409:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.636410:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.636415:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.636422:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.636424:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.636425:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.636431:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.636433:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.636434:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.636436:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636441:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.636446:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.636448:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.636450:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.636452:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116705, 116705] 00000080:00200000:1.0:1788201706.636459:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.636461:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.636478:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.636480:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.636490:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478027776, count: 4096 00000080:00200000:1.0:1788201706.636496:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478027776, bytes: 4096 00000080:00200000:1.0:1788201706.636497:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.636500:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.636501:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478027776, 478031872) 0 0 00000080:00200000:1.0:1788201706.636502:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.636505:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116706, 116706] 00020000:00200000:1.0:1788201706.636507:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478027776 chunk: [478027776, 478031872) 478031872, 4096 00020000:00200000:1.0:1788201706.636508:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.636515:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116706, 116706] 00000008:00010000:1.0:1788201706.636518:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.636519:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.636524:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636530:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478027776 478031871) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636535:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636539:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636547:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.636552:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478027776, 478031872) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.636554:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116706 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.636558:0:17463:0:(rw26.c:898:ll_write_end()) pos 478027776, len 4096, copied 4096 00000080:00200000:1.0:1788201706.636560:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d4671458[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116706] 00000080:00200000:1.0:1788201706.636561:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bc2b2d0d fffffc0001029 2:0 ffff93c0c1221620 116706 lru 00000080:00200000:1.0:1788201706.636565:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000052c17f41 116706: 1< 2 - - > 2< 478027776 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68291 - -> 00000080:00200000:1.0:1788201706.636567:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d4671458 00000080:00200000:1.0:1788201706.636567:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.636569:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.636574:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.636581:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.636583:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.636585:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.636590:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.636592:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.636593:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.636596:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636601:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.636610:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.636611:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.636614:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.636615:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116706, 116706] 00000080:00200000:1.0:1788201706.636622:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.636624:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.636641:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.636644:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.636653:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478031872, count: 4096 00000080:00200000:1.0:1788201706.636659:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478031872, bytes: 4096 00000080:00200000:1.0:1788201706.636661:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.636663:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.636664:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478031872, 478035968) 0 0 00000080:00200000:1.0:1788201706.636666:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.636669:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116707, 116707] 00020000:00200000:1.0:1788201706.636671:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478031872 chunk: [478031872, 478035968) 478035968, 4096 00020000:00200000:1.0:1788201706.636672:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.636678:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116707, 116707] 00000008:00010000:1.0:1788201706.636681:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.636684:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.636691:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636699:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478031872 478035967) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636708:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636720:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636728:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.636735:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478031872, 478035968) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.636741:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116707 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.636746:0:17463:0:(rw26.c:898:ll_write_end()) pos 478031872, len 4096, copied 4096 00000080:00200000:1.0:1788201706.636750:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000be242c8[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116707] 00000080:00200000:1.0:1788201706.636752:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e46f0c6d fffffc0001029 2:0 ffff93c0c1221d20 116707 lru 00000080:00200000:1.0:1788201706.636758:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000001a2e2ec 116707: 1< 2 - - > 2< 478031872 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68292 - -> 00000080:00200000:1.0:1788201706.636760:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000be242c8 00000080:00200000:1.0:1788201706.636761:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.636763:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.636772:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.636785:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.636788:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.636790:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.636799:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.636800:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.636802:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.636805:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636814:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.636819:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.636820:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.636822:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.636824:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116707, 116707] 00000080:00200000:1.0:1788201706.636845:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.636848:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.636867:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.636869:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.636880:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478035968, count: 4096 00000080:00200000:1.0:1788201706.636886:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478035968, bytes: 4096 00000080:00200000:1.0:1788201706.636887:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.636890:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.636891:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478035968, 478040064) 0 0 00000080:00200000:1.0:1788201706.636892:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.636895:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116708, 116708] 00020000:00200000:1.0:1788201706.636898:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478035968 chunk: [478035968, 478040064) 478040064, 4096 00020000:00200000:1.0:1788201706.636899:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.636905:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116708, 116708] 00000008:00010000:1.0:1788201706.636909:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.636910:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.636915:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636921:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478035968 478040063) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636930:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636934:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636938:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.636943:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478035968, 478040064) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.636946:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116708 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.636950:0:17463:0:(rw26.c:898:ll_write_end()) pos 478035968, len 4096, copied 4096 00000080:00200000:1.0:1788201706.636952:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f9e38fd0[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116708] 00000080:00200000:1.0:1788201706.636954:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007569704f fffffc0001029 2:0 ffff93c0c12219a0 116708 lru 00000080:00200000:1.0:1788201706.636957:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000052a41ecf 116708: 1< 2 - - > 2< 478035968 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68293 - -> 00000080:00200000:1.0:1788201706.636959:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f9e38fd0 00000080:00200000:1.0:1788201706.636960:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.636961:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.636967:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.636974:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.636976:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.636977:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.636983:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.636985:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.636987:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.636989:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.636994:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.636999:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.637000:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.637002:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.637004:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116708, 116708] 00000080:00200000:1.0:1788201706.637011:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.637013:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.637043:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.637046:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.637056:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478040064, count: 4096 00000080:00200000:1.0:1788201706.637062:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478040064, bytes: 4096 00000080:00200000:1.0:1788201706.637064:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.637066:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.637067:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478040064, 478044160) 0 0 00000080:00200000:1.0:1788201706.637068:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.637071:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116709, 116709] 00020000:00200000:1.0:1788201706.637073:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478040064 chunk: [478040064, 478044160) 478044160, 4096 00020000:00200000:1.0:1788201706.637075:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.637081:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116709, 116709] 00000008:00010000:1.0:1788201706.637084:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.637085:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.637091:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637100:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478040064 478044159) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637105:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637110:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637114:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.637118:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478040064, 478044160) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.637121:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116709 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.637125:0:17463:0:(rw26.c:898:ll_write_end()) pos 478040064, len 4096, copied 4096 00000080:00200000:1.0:1788201706.637127:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b0815879[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116709] 00000080:00200000:1.0:1788201706.637129:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005e43d4da fffffc0001029 2:0 ffff93c0c1221b60 116709 lru 00000080:00200000:1.0:1788201706.637132:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000048ab5910 116709: 1< 2 - - > 2< 478040064 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68294 - -> 00000080:00200000:1.0:1788201706.637134:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b0815879 00000080:00200000:1.0:1788201706.637135:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.637136:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.637141:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.637149:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.637151:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.637152:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.637158:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.637160:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.637161:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.637164:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637169:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.637174:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.637175:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.637177:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.637179:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116709, 116709] 00000080:00200000:1.0:1788201706.637185:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.637187:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.637205:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.637207:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.637216:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478044160, count: 4096 00000080:00200000:1.0:1788201706.637222:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478044160, bytes: 4096 00000080:00200000:1.0:1788201706.637224:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.637226:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.637227:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478044160, 478048256) 0 0 00000080:00200000:1.0:1788201706.637229:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.637232:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116710, 116710] 00020000:00200000:1.0:1788201706.637234:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478044160 chunk: [478044160, 478048256) 478048256, 4096 00020000:00200000:1.0:1788201706.637235:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.637241:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116710, 116710] 00000008:00010000:1.0:1788201706.637244:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.637246:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.637251:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637256:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478044160 478048255) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637262:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637266:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637270:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.637274:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478044160, 478048256) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.637277:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116710 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.637280:0:17463:0:(rw26.c:898:ll_write_end()) pos 478044160, len 4096, copied 4096 00000080:00200000:1.0:1788201706.637282:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000030a494ff[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116710] 00000080:00200000:1.0:1788201706.637284:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006e12f2f3 fffffc0001029 2:0 ffff93c0c1221c40 116710 lru 00000080:00200000:1.0:1788201706.637287:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000127aa8d5 116710: 1< 2 - - > 2< 478044160 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68295 - -> 00000080:00200000:1.0:1788201706.637288:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000030a494ff 00000080:00200000:1.0:1788201706.637289:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.637290:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.637295:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.637303:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.637305:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.637306:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.637312:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.637314:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.637315:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.637318:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637323:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.637328:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.637329:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.637331:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.637333:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116710, 116710] 00000080:00200000:1.0:1788201706.637340:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.637341:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.637362:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.637366:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.637380:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478048256, count: 4096 00000080:00200000:1.0:1788201706.637391:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478048256, bytes: 4096 00000080:00200000:1.0:1788201706.637394:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.637398:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.637400:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478048256, 478052352) 0 0 00000080:00200000:1.0:1788201706.637402:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.637407:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116711, 116711] 00020000:00200000:1.0:1788201706.637410:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478048256 chunk: [478048256, 478052352) 478052352, 4096 00020000:00200000:1.0:1788201706.637412:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.637423:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116711, 116711] 00000008:00010000:1.0:1788201706.637426:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.637428:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.637434:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637439:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478048256 478052351) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637445:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637449:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637453:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.637458:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478048256, 478052352) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.637461:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116711 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.637465:0:17463:0:(rw26.c:898:ll_write_end()) pos 478048256, len 4096, copied 4096 00000080:00200000:1.0:1788201706.637468:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000080234a39[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116711] 00000080:00200000:1.0:1788201706.637470:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bf49363b fffffc0001029 2:0 ffff93c0c1221540 116711 lru 00000080:00200000:1.0:1788201706.637473:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006c440898 116711: 1< 2 - - > 2< 478048256 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68296 - -> 00000080:00200000:1.0:1788201706.637475:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000080234a39 00000080:00200000:1.0:1788201706.637476:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.637477:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.637483:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.637494:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.637495:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.637497:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.637503:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.637505:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.637507:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.637509:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637514:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.637520:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.637522:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.637525:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.637528:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116711, 116711] 00000080:00200000:1.0:1788201706.637538:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.637541:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.637569:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.637573:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.637587:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478052352, count: 4096 00000080:00200000:1.0:1788201706.637597:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478052352, bytes: 4096 00000080:00200000:1.0:1788201706.637599:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.637602:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.637604:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478052352, 478056448) 0 0 00000080:00200000:1.0:1788201706.637606:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.637610:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116712, 116712] 00020000:00200000:1.0:1788201706.637613:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478052352 chunk: [478052352, 478056448) 478056448, 4096 00020000:00200000:1.0:1788201706.637615:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.637625:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116712, 116712] 00000008:00010000:1.0:1788201706.637630:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.637632:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.637639:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637647:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478052352 478056447) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637655:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637662:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637668:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.637676:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478052352, 478056448) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.637680:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116712 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.637686:0:17463:0:(rw26.c:898:ll_write_end()) pos 478052352, len 4096, copied 4096 00000080:00200000:1.0:1788201706.637689:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006f7b7f16[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116712] 00000080:00200000:1.0:1788201706.637692:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009d0ca210 fffffc0001029 2:0 ffff93c0c1221460 116712 lru 00000080:00200000:1.0:1788201706.637697:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d93812ac 116712: 1< 2 - - > 2< 478052352 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68297 - -> 00000080:00200000:1.0:1788201706.637699:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006f7b7f16 00000080:00200000:1.0:1788201706.637700:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.637702:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.637711:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.637723:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.637726:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.637728:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.637737:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.637740:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.637741:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.637744:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637750:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.637757:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.637759:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.637763:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.637766:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116712, 116712] 00000080:00200000:1.0:1788201706.637777:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.637781:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.637812:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.637816:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.637850:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478056448, count: 4096 00000080:00200000:1.0:1788201706.637862:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478056448, bytes: 4096 00000080:00200000:1.0:1788201706.637865:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.637869:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.637870:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478056448, 478060544) 0 0 00000080:00200000:1.0:1788201706.637873:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.637878:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116713, 116713] 00020000:00200000:1.0:1788201706.637882:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478056448 chunk: [478056448, 478060544) 478060544, 4096 00020000:00200000:1.0:1788201706.637884:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.637896:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116713, 116713] 00000008:00010000:1.0:1788201706.637901:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.637904:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.637912:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637921:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478056448 478060543) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637930:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637938:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.637945:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.637953:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478056448, 478060544) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.637958:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116713 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.637964:0:17463:0:(rw26.c:898:ll_write_end()) pos 478056448, len 4096, copied 4096 00000080:00200000:1.0:1788201706.637968:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002a7c564f[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116713] 00000080:00200000:1.0:1788201706.637971:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000045818459 fffffc0001029 2:0 ffff93c0c12210e0 116713 lru 00000080:00200000:1.0:1788201706.637977:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005464a3db 116713: 1< 2 - - > 2< 478056448 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68298 - -> 00000080:00200000:1.0:1788201706.637980:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002a7c564f 00000080:00200000:1.0:1788201706.637981:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.637983:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.637992:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.638004:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.638008:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.638010:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.638034:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.638038:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.638040:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.638045:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638053:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.638062:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.638064:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.638069:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.638072:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116713, 116713] 00000080:00200000:1.0:1788201706.638084:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.638088:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.638117:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.638121:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.638137:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478060544, count: 4096 00000080:00200000:1.0:1788201706.638149:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478060544, bytes: 4096 00000080:00200000:1.0:1788201706.638152:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.638155:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.638157:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478060544, 478064640) 0 0 00000080:00200000:1.0:1788201706.638159:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.638165:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116714, 116714] 00020000:00200000:1.0:1788201706.638168:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478060544 chunk: [478060544, 478064640) 478064640, 4096 00020000:00200000:1.0:1788201706.638170:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.638182:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116714, 116714] 00000008:00010000:1.0:1788201706.638188:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.638190:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.638198:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638207:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478060544 478064639) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638216:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638223:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638230:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.638238:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478060544, 478064640) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.638243:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116714 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.638248:0:17463:0:(rw26.c:898:ll_write_end()) pos 478060544, len 4096, copied 4096 00000080:00200000:1.0:1788201706.638253:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ee1b4204[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116714] 00000080:00200000:1.0:1788201706.638256:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003c24871b fffffc0001029 2:0 ffff93c0c1221700 116714 lru 00000080:00200000:1.0:1788201706.638262:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000001d9f7e6 116714: 1< 2 - - > 2< 478060544 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68299 - -> 00000080:00200000:1.0:1788201706.638264:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ee1b4204 00000080:00200000:1.0:1788201706.638265:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.638267:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.638276:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.638288:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.638291:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.638293:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.638303:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.638306:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.638309:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.638313:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638321:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.638330:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.638332:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.638336:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.638340:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116714, 116714] 00000080:00200000:1.0:1788201706.638350:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.638354:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.638385:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.638388:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.638406:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478064640, count: 4096 00000080:00200000:1.0:1788201706.638417:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478064640, bytes: 4096 00000080:00200000:1.0:1788201706.638420:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.638424:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.638425:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478064640, 478068736) 0 0 00000080:00200000:1.0:1788201706.638427:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.638433:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116715, 116715] 00020000:00200000:1.0:1788201706.638436:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478064640 chunk: [478064640, 478068736) 478068736, 4096 00020000:00200000:1.0:1788201706.638439:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.638451:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116715, 116715] 00000008:00010000:1.0:1788201706.638457:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.638459:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.638467:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638476:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478064640 478068735) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638485:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638493:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638500:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.638507:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478064640, 478068736) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.638512:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116715 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.638518:0:17463:0:(rw26.c:898:ll_write_end()) pos 478064640, len 4096, copied 4096 00000080:00200000:1.0:1788201706.638522:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000034bce66d[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116715] 00000080:00200000:1.0:1788201706.638525:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000427cc5e2 fffffc0001029 2:0 ffff93c0c1221000 116715 lru 00000080:00200000:1.0:1788201706.638530:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000749d0bd0 116715: 1< 2 - - > 2< 478064640 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68300 - -> 00000080:00200000:1.0:1788201706.638533:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000034bce66d 00000080:00200000:1.0:1788201706.638534:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.638536:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.638544:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.638556:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.638559:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.638561:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.638570:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.638573:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.638575:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.638579:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638587:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.638595:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.638597:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.638601:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.638604:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116715, 116715] 00000080:00200000:1.0:1788201706.638615:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.638619:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.638649:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.638653:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.638671:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478068736, count: 4096 00000080:00200000:1.0:1788201706.638682:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478068736, bytes: 4096 00000080:00200000:1.0:1788201706.638685:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.638689:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.638691:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478068736, 478072832) 0 0 00000080:00200000:1.0:1788201706.638693:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.638698:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116716, 116716] 00020000:00200000:1.0:1788201706.638702:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478068736 chunk: [478068736, 478072832) 478072832, 4096 00020000:00200000:1.0:1788201706.638704:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.638715:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116716, 116716] 00000008:00010000:1.0:1788201706.638721:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.638723:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.638730:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638739:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478068736 478072831) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638748:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638756:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638763:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.638771:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478068736, 478072832) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.638777:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116716 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.638782:0:17463:0:(rw26.c:898:ll_write_end()) pos 478068736, len 4096, copied 4096 00000080:00200000:1.0:1788201706.638786:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008615c594[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116716] 00000080:00200000:1.0:1788201706.638789:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ae7946c6 fffffc0001029 2:0 ffff93c0c1221a80 116716 lru 00000080:00200000:1.0:1788201706.638794:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000007422bc6 116716: 1< 2 - - > 2< 478068736 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68301 - -> 00000080:00200000:1.0:1788201706.638797:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008615c594 00000080:00200000:1.0:1788201706.638798:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.638800:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.638809:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.638821:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.638824:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.638826:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.638853:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.638856:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.638859:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.638863:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.638871:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.638880:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.638882:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.638886:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.638889:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116716, 116716] 00000080:00200000:1.0:1788201706.638900:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.638904:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.638933:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.638937:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.638954:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478072832, count: 4096 00000080:00200000:1.0:1788201706.638965:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478072832, bytes: 4096 00000080:00200000:1.0:1788201706.638967:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.638971:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.638972:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478072832, 478076928) 0 0 00000080:00200000:1.0:1788201706.638975:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.638980:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116717, 116717] 00020000:00200000:1.0:1788201706.638983:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478072832 chunk: [478072832, 478076928) 478076928, 4096 00020000:00200000:1.0:1788201706.638985:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.638997:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116717, 116717] 00000008:00010000:1.0:1788201706.639002:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.639004:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.639012:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639036:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478072832 478076927) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639045:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639053:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639060:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.639073:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478072832, 478076928) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.639078:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116717 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.639083:0:17463:0:(rw26.c:898:ll_write_end()) pos 478072832, len 4096, copied 4096 00000080:00200000:1.0:1788201706.639087:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006c8f7238[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116717] 00000080:00200000:1.0:1788201706.639090:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ceac85fb fffffc0001029 2:0 ffff93c0c1221380 116717 lru 00000080:00200000:1.0:1788201706.639096:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008ac24152 116717: 1< 2 - - > 2< 478072832 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68302 - -> 00000080:00200000:1.0:1788201706.639098:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006c8f7238 00000080:00200000:1.0:1788201706.639099:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.639101:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.639110:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.639122:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.639125:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.639127:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.639136:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.639138:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.639141:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.639145:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639154:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.639163:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.639165:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.639169:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.639172:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116717, 116717] 00000080:00200000:1.0:1788201706.639179:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.639182:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.639201:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.639203:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.639216:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478076928, count: 4096 00000080:00200000:1.0:1788201706.639226:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478076928, bytes: 4096 00000080:00200000:1.0:1788201706.639228:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.639232:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.639233:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478076928, 478081024) 0 0 00000080:00200000:1.0:1788201706.639236:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.639241:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116718, 116718] 00020000:00200000:1.0:1788201706.639244:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478076928 chunk: [478076928, 478081024) 478081024, 4096 00020000:00200000:1.0:1788201706.639246:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.639257:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116718, 116718] 00000008:00010000:1.0:1788201706.639262:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.639264:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.639271:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639277:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478076928 478081023) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639283:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639287:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639295:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.639300:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478076928, 478081024) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.639303:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116718 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.639307:0:17463:0:(rw26.c:898:ll_write_end()) pos 478076928, len 4096, copied 4096 00000080:00200000:1.0:1788201706.639309:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004b4700d5[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116718] 00000080:00200000:1.0:1788201706.639311:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000018813dbf fffffc0001029 2:0 ffff93c0c12218c0 116718 lru 00000080:00200000:1.0:1788201706.639315:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000039d8683c 116718: 1< 2 - - > 2< 478076928 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68303 - -> 00000080:00200000:1.0:1788201706.639316:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004b4700d5 00000080:00200000:1.0:1788201706.639317:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.639318:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.639324:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.639332:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.639333:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.639335:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.639341:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.639343:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.639345:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.639349:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639357:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.639370:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.639373:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.639377:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.639380:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116718, 116718] 00000080:00200000:1.0:1788201706.639391:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.639394:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.639416:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.639419:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.639429:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478081024, count: 4096 00000080:00200000:1.0:1788201706.639436:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478081024, bytes: 4096 00000080:00200000:1.0:1788201706.639437:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.639440:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.639441:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478081024, 478085120) 0 0 00000080:00200000:1.0:1788201706.639442:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.639446:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116719, 116719] 00020000:00200000:1.0:1788201706.639447:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478081024 chunk: [478081024, 478085120) 478085120, 4096 00020000:00200000:1.0:1788201706.639449:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.639456:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116719, 116719] 00000008:00010000:1.0:1788201706.639459:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.639460:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.639466:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639471:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478081024 478085119) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639477:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639485:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639490:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.639494:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478081024, 478085120) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.639496:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116719 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.639500:0:17463:0:(rw26.c:898:ll_write_end()) pos 478081024, len 4096, copied 4096 00000080:00200000:1.0:1788201706.639502:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000012749222[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116719] 00000080:00200000:1.0:1788201706.639504:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004e4d4745 fffffc0001029 2:0 ffff93c0c1221ee0 116719 lru 00000080:00200000:1.0:1788201706.639507:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000074e016a4 116719: 1< 2 - - > 2< 478081024 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68304 - -> 00000080:00200000:1.0:1788201706.639509:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000012749222 00000080:00200000:1.0:1788201706.639509:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.639510:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.639517:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.639524:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.639526:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.639527:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.639533:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.639535:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.639537:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.639539:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639547:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.639552:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.639553:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.639555:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.639557:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116719, 116719] 00000080:00200000:1.0:1788201706.639564:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.639566:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.639583:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.639586:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.639596:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478085120, count: 4096 00000080:00200000:1.0:1788201706.639602:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478085120, bytes: 4096 00000080:00200000:1.0:1788201706.639603:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.639606:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.639607:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478085120, 478089216) 0 0 00000080:00200000:1.0:1788201706.639608:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.639611:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116720, 116720] 00020000:00200000:1.0:1788201706.639613:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478085120 chunk: [478085120, 478089216) 478089216, 4096 00020000:00200000:1.0:1788201706.639615:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.639621:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116720, 116720] 00000008:00010000:1.0:1788201706.639624:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.639625:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.639630:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639636:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478085120 478089215) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639645:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639650:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639653:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.639658:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478085120, 478089216) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.639661:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116720 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.639665:0:17463:0:(rw26.c:898:ll_write_end()) pos 478085120, len 4096, copied 4096 00000080:00200000:1.0:1788201706.639667:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002afbee0f[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116720] 00000080:00200000:1.0:1788201706.639669:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005f79a06f fffffc0001029 2:0 ffff93c0c12217e0 116720 lru 00000080:00200000:1.0:1788201706.639673:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004fc6462d 116720: 1< 2 - - > 2< 478085120 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68305 - -> 00000080:00200000:1.0:1788201706.639674:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002afbee0f 00000080:00200000:1.0:1788201706.639675:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.639676:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.639682:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.639689:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.639691:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.639692:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.639698:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.639700:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.639702:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.639704:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639709:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.639714:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.639715:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.639717:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.639719:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116720, 116720] 00000080:00200000:1.0:1788201706.639726:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.639728:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.639745:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.639747:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.639757:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478089216, count: 4096 00000080:00200000:1.0:1788201706.639763:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478089216, bytes: 4096 00000080:00200000:1.0:1788201706.639765:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.639767:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.639768:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478089216, 478093312) 0 0 00000080:00200000:1.0:1788201706.639770:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.639773:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116721, 116721] 00020000:00200000:1.0:1788201706.639775:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478089216 chunk: [478089216, 478093312) 478093312, 4096 00020000:00200000:1.0:1788201706.639776:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.639782:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116721, 116721] 00000008:00010000:1.0:1788201706.639785:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.639787:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.639792:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639801:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478089216 478093311) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639806:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639811:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639815:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.639819:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478089216, 478093312) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.639822:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116721 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.639826:0:17463:0:(rw26.c:898:ll_write_end()) pos 478089216, len 4096, copied 4096 00000080:00200000:1.0:1788201706.639828:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000783e6e2a[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116721] 00000080:00200000:1.0:1788201706.639843:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c173fa01 fffffc0001029 2:0 ffff93c0c12212a0 116721 lru 00000080:00200000:1.0:1788201706.639847:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003ee0bd97 116721: 1< 2 - - > 2< 478089216 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68306 - -> 00000080:00200000:1.0:1788201706.639848:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000783e6e2a 00000080:00200000:1.0:1788201706.639849:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.639850:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.639856:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.639863:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.639865:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.639867:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.639873:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.639874:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.639876:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.639878:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639883:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.639888:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.639889:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.639892:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.639893:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116721, 116721] 00000080:00200000:1.0:1788201706.639900:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.639902:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.639920:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.639922:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.639932:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478093312, count: 4096 00000080:00200000:1.0:1788201706.639938:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478093312, bytes: 4096 00000080:00200000:1.0:1788201706.639939:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.639941:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.639942:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478093312, 478097408) 0 0 00000080:00200000:1.0:1788201706.639944:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.639947:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116722, 116722] 00020000:00200000:1.0:1788201706.639949:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478093312 chunk: [478093312, 478097408) 478097408, 4096 00020000:00200000:1.0:1788201706.639950:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.639956:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116722, 116722] 00000008:00010000:1.0:1788201706.639959:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.639961:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.639966:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639972:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478093312 478097407) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639977:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639981:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.639985:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.639990:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478093312, 478097408) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.639993:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116722 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.639997:0:17463:0:(rw26.c:898:ll_write_end()) pos 478093312, len 4096, copied 4096 00000080:00200000:1.0:1788201706.639999:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bb1957bb[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116722] 00000080:00200000:1.0:1788201706.640001:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009bd65ac0 fffffc0001029 2:0 ffff93c0c1221e00 116722 lru 00000080:00200000:1.0:1788201706.640004:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000042a8b3b7 116722: 1< 2 - - > 2< 478093312 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68307 - -> 00000080:00200000:1.0:1788201706.640006:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bb1957bb 00000080:00200000:1.0:1788201706.640006:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.640008:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.640013:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.640036:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.640038:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.640040:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.640047:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.640048:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.640050:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.640053:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640058:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.640063:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.640064:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.640067:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.640069:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116722, 116722] 00000080:00200000:1.0:1788201706.640076:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.640077:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.640101:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.640105:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.640122:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478097408, count: 4096 00000080:00200000:1.0:1788201706.640131:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478097408, bytes: 4096 00000080:00200000:1.0:1788201706.640134:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.640137:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.640138:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478097408, 478101504) 0 0 00000080:00200000:1.0:1788201706.640140:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.640144:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116723, 116723] 00020000:00200000:1.0:1788201706.640147:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478097408 chunk: [478097408, 478101504) 478101504, 4096 00020000:00200000:1.0:1788201706.640149:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.640159:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116723, 116723] 00000008:00010000:1.0:1788201706.640164:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.640166:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.640174:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640182:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478097408 478101503) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640190:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640196:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640201:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.640208:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478097408, 478101504) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.640213:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116723 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.640219:0:17463:0:(rw26.c:898:ll_write_end()) pos 478097408, len 4096, copied 4096 00000080:00200000:1.0:1788201706.640223:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b0297ed3[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116723] 00000080:00200000:1.0:1788201706.640225:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000039675a0e fffffc0001029 2:0 ffff93c0c12211c0 116723 lru 00000080:00200000:1.0:1788201706.640230:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f1a2aa0c 116723: 1< 2 - - > 2< 478097408 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68308 - -> 00000080:00200000:1.0:1788201706.640231:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b0297ed3 00000080:00200000:1.0:1788201706.640232:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.640234:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.640242:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.640257:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.640260:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.640262:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.640270:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.640272:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.640274:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.640278:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640284:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.640292:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.640294:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.640298:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.640300:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116723, 116723] 00000080:00200000:1.0:1788201706.640311:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.640314:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.640340:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.640343:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.640367:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478101504, count: 4096 00000080:00200000:1.0:1788201706.640383:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478101504, bytes: 4096 00000080:00200000:1.0:1788201706.640387:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.640390:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.640392:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478101504, 478105600) 0 0 00000080:00200000:1.0:1788201706.640394:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.640399:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116724, 116724] 00020000:00200000:1.0:1788201706.640402:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478101504 chunk: [478101504, 478105600) 478105600, 4096 00020000:00200000:1.0:1788201706.640403:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.640411:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116724, 116724] 00000008:00010000:1.0:1788201706.640415:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.640416:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.640422:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640428:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478101504 478105599) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640433:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640438:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640442:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.640447:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478101504, 478105600) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.640450:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116724 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.640454:0:17463:0:(rw26.c:898:ll_write_end()) pos 478101504, len 4096, copied 4096 00000080:00200000:1.0:1788201706.640456:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e6529d93[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116724] 00000080:00200000:1.0:1788201706.640458:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000019e82ccd fffffc0001029 2:0 ffff93c0c1222e00 116724 lru 00000080:00200000:1.0:1788201706.640462:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005fe6ae55 116724: 1< 2 - - > 2< 478101504 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68309 - -> 00000080:00200000:1.0:1788201706.640463:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e6529d93 00000080:00200000:1.0:1788201706.640464:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.640465:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.640471:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.640480:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.640481:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.640483:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.640489:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.640491:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.640493:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.640495:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640500:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.640505:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.640506:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.640508:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.640510:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116724, 116724] 00000080:00200000:1.0:1788201706.640517:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.640519:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.640537:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.640539:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.640550:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478105600, count: 4096 00000080:00200000:1.0:1788201706.640556:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478105600, bytes: 4096 00000080:00200000:1.0:1788201706.640557:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.640560:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.640561:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478105600, 478109696) 0 0 00000080:00200000:1.0:1788201706.640563:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.640566:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116725, 116725] 00020000:00200000:1.0:1788201706.640568:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478105600 chunk: [478105600, 478109696) 478109696, 4096 00020000:00200000:1.0:1788201706.640569:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.640575:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116725, 116725] 00000008:00010000:1.0:1788201706.640578:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.640580:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.640584:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640590:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478105600 478109695) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640595:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640600:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640604:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.640608:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478105600, 478109696) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.640610:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116725 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.640614:0:17463:0:(rw26.c:898:ll_write_end()) pos 478105600, len 4096, copied 4096 00000080:00200000:1.0:1788201706.640616:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000dfedda96[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116725] 00000080:00200000:1.0:1788201706.640617:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000055d4d16a fffffc0001029 2:0 ffff93c0c12221c0 116725 lru 00000080:00200000:1.0:1788201706.640621:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008fee002b 116725: 1< 2 - - > 2< 478105600 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68310 - -> 00000080:00200000:1.0:1788201706.640623:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000dfedda96 00000080:00200000:1.0:1788201706.640624:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.640626:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.640631:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.640639:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.640641:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.640642:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.640648:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.640649:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.640651:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.640653:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640658:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.640663:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.640664:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.640666:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.640668:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116725, 116725] 00000080:00200000:1.0:1788201706.640674:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.640676:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.640694:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.640696:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.640706:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478109696, count: 4096 00000080:00200000:1.0:1788201706.640712:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478109696, bytes: 4096 00000080:00200000:1.0:1788201706.640714:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.640716:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.640717:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478109696, 478113792) 0 0 00000080:00200000:1.0:1788201706.640718:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.640722:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116726, 116726] 00020000:00200000:1.0:1788201706.640723:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478109696 chunk: [478109696, 478113792) 478113792, 4096 00020000:00200000:1.0:1788201706.640725:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.640731:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116726, 116726] 00000008:00010000:1.0:1788201706.640734:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.640735:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.640740:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640746:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478109696 478113791) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640751:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640756:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640760:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.640764:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478109696, 478113792) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.640767:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116726 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.640770:0:17463:0:(rw26.c:898:ll_write_end()) pos 478109696, len 4096, copied 4096 00000080:00200000:1.0:1788201706.640772:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000eb729901[3 0000000075f4bf38 2 1 00000000d49d63ed 0 0 116726] 00000080:00200000:1.0:1788201706.640774:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d918f1d6 fffffc0001029 2:0 ffff93c0c1222620 116726 lru 00000080:00200000:1.0:1788201706.640778:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000025711c5a 116726: 1< 2 - - > 2< 478109696 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68311 - -> 00000080:00200000:1.0:1788201706.640779:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000eb729901 00000080:00200000:1.0:1788201706.640780:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.640781:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.640787:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.640793:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.640795:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.640796:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.640802:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.640804:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.640805:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.640808:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.640814:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.640822:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1beef38 (ffff93c1d1bf11b8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.640824:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf11b8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.640860:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.640865:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116726, 116726] 00000080:00200000:1.0:1788201706.640876:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.640880:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.640921:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.640927:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.640952:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478113792, count: 4096 00000080:00200000:0.0:1788201706.640972:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478113792, bytes: 4096 00000080:00200000:0.0:1788201706.640976:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.640980:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.640982:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478113792, 478117888) 0 0 00000080:00200000:0.0:1788201706.640985:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.640991:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116727, 116727] 00020000:00200000:0.0:1788201706.640994:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478113792 chunk: [478113792, 478117888) 478117888, 4096 00020000:00200000:0.0:1788201706.640995:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.641013:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116727, 116727] 00000008:00010000:0.0:1788201706.641034:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.641037:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.641046:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641058:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478113792 478117887) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641066:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641071:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641075:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.641080:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478113792, 478117888) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.641085:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116727 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.641089:0:17463:0:(rw26.c:898:ll_write_end()) pos 478113792, len 4096, copied 4096 00000080:00200000:0.0:1788201706.641093:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ba7692a0[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116727] 00000080:00200000:0.0:1788201706.641095:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006d1e5376 fffffc0001029 2:0 ffff93c0c1222d20 116727 lru 00000080:00200000:0.0:1788201706.641099:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d2100b76 116727: 1< 2 - - > 2< 478113792 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68312 - -> 00000080:00200000:0.0:1788201706.641100:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ba7692a0 00000080:00200000:0.0:1788201706.641101:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.641102:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.641110:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.641124:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.641126:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.641128:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.641137:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.641139:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.641141:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.641144:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641149:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.641155:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.641156:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.641160:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.641163:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116727, 116727] 00000080:00200000:0.0:1788201706.641176:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.641178:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.641210:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.641215:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.641233:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478117888, count: 4096 00000080:00200000:0.0:1788201706.641245:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478117888, bytes: 4096 00000080:00200000:0.0:1788201706.641248:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.641252:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.641253:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478117888, 478121984) 0 0 00000080:00200000:0.0:1788201706.641256:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.641261:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116728, 116728] 00020000:00200000:0.0:1788201706.641264:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478117888 chunk: [478117888, 478121984) 478121984, 4096 00020000:00200000:0.0:1788201706.641266:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.641275:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116728, 116728] 00000008:00010000:0.0:1788201706.641285:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.641287:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.641293:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641300:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478117888 478121983) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641305:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641310:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641314:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.641319:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478117888, 478121984) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.641323:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116728 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.641327:0:17463:0:(rw26.c:898:ll_write_end()) pos 478117888, len 4096, copied 4096 00000080:00200000:0.0:1788201706.641330:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ff83389a[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116728] 00000080:00200000:0.0:1788201706.641332:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e136b163 fffffc0001029 2:0 ffff93c0c12229a0 116728 lru 00000080:00200000:0.0:1788201706.641336:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d64746db 116728: 1< 2 - - > 2< 478117888 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68313 - -> 00000080:00200000:0.0:1788201706.641337:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ff83389a 00000080:00200000:0.0:1788201706.641338:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.641340:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.641346:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.641355:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.641357:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.641358:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.641365:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.641367:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.641369:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.641371:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641376:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.641381:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.641383:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.641385:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.641387:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116728, 116728] 00000080:00200000:0.0:1788201706.641395:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.641397:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.641416:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.641419:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.641429:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478121984, count: 4096 00000080:00200000:0.0:1788201706.641435:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478121984, bytes: 4096 00000080:00200000:0.0:1788201706.641437:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.641440:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.641441:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478121984, 478126080) 0 0 00000080:00200000:0.0:1788201706.641443:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.641446:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116729, 116729] 00020000:00200000:0.0:1788201706.641448:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478121984 chunk: [478121984, 478126080) 478126080, 4096 00020000:00200000:0.0:1788201706.641450:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.641457:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116729, 116729] 00000008:00010000:0.0:1788201706.641461:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.641462:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.641467:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641473:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478121984 478126079) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641479:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641484:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641487:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.641492:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478121984, 478126080) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.641495:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116729 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.641500:0:17463:0:(rw26.c:898:ll_write_end()) pos 478121984, len 4096, copied 4096 00000080:00200000:0.0:1788201706.641502:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f063724a[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116729] 00000080:00200000:0.0:1788201706.641504:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004487c919 fffffc0001029 2:0 ffff93c0c1222b60 116729 lru 00000080:00200000:0.0:1788201706.641508:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001f9ca217 116729: 1< 2 - - > 2< 478121984 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68314 - -> 00000080:00200000:0.0:1788201706.641511:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f063724a 00000080:00200000:0.0:1788201706.641512:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.641513:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.641519:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.641527:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.641528:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.641530:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.641536:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.641538:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.641539:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.641542:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641546:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.641551:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.641553:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.641556:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.641558:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116729, 116729] 00000080:00200000:0.0:1788201706.641565:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.641567:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.641594:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.641597:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.641608:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478126080, count: 4096 00000080:00200000:0.0:1788201706.641615:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478126080, bytes: 4096 00000080:00200000:0.0:1788201706.641617:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.641620:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.641621:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478126080, 478130176) 0 0 00000080:00200000:0.0:1788201706.641622:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.641625:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116730, 116730] 00020000:00200000:0.0:1788201706.641627:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478126080 chunk: [478126080, 478130176) 478130176, 4096 00020000:00200000:0.0:1788201706.641629:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.641636:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116730, 116730] 00000008:00010000:0.0:1788201706.641639:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.641641:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.641646:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641652:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478126080 478130175) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641657:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641662:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641665:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.641670:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478126080, 478130176) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.641673:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116730 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.641677:0:17463:0:(rw26.c:898:ll_write_end()) pos 478126080, len 4096, copied 4096 00000080:00200000:0.0:1788201706.641680:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008e62b6ab[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116730] 00000080:00200000:0.0:1788201706.641682:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bab696e3 fffffc0001029 2:0 ffff93c0c1222c40 116730 lru 00000080:00200000:0.0:1788201706.641686:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006e5030e7 116730: 1< 2 - - > 2< 478126080 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68315 - -> 00000080:00200000:0.0:1788201706.641688:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008e62b6ab 00000080:00200000:0.0:1788201706.641688:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.641690:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.641695:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.641702:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.641704:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.641706:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.641712:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.641713:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.641715:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.641717:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641722:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.641727:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.641728:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.641731:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.641733:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116730, 116730] 00000080:00200000:0.0:1788201706.641740:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.641742:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.641760:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.641762:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.641772:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478130176, count: 4096 00000080:00200000:0.0:1788201706.641778:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478130176, bytes: 4096 00000080:00200000:0.0:1788201706.641780:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.641782:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.641790:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478130176, 478134272) 0 0 00000080:00200000:0.0:1788201706.641792:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.641798:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116731, 116731] 00020000:00200000:0.0:1788201706.641800:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478130176 chunk: [478130176, 478134272) 478134272, 4096 00020000:00200000:0.0:1788201706.641802:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.641809:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116731, 116731] 00000008:00010000:0.0:1788201706.641812:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.641813:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.641818:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641824:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478130176 478134271) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641844:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641849:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641854:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.641858:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478130176, 478134272) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.641861:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116731 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.641865:0:17463:0:(rw26.c:898:ll_write_end()) pos 478130176, len 4096, copied 4096 00000080:00200000:0.0:1788201706.641868:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fd593760[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116731] 00000080:00200000:0.0:1788201706.641869:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ed412e73 fffffc0001029 2:0 ffff93c0c1222540 116731 lru 00000080:00200000:0.0:1788201706.641873:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000162d4c96 116731: 1< 2 - - > 2< 478130176 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68316 - -> 00000080:00200000:0.0:1788201706.641875:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fd593760 00000080:00200000:0.0:1788201706.641875:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.641877:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.641883:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.641891:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.641892:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.641894:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.641900:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.641902:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.641903:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.641905:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.641910:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.641914:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.641916:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.641918:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.641920:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116731, 116731] 00000080:00200000:0.0:1788201706.641927:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.641929:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.641947:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.641949:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.641959:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478134272, count: 4096 00000080:00200000:0.0:1788201706.641965:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478134272, bytes: 4096 00000080:00200000:0.0:1788201706.641967:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.641969:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.641970:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478134272, 478138368) 0 0 00000080:00200000:0.0:1788201706.641972:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.641975:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116732, 116732] 00020000:00200000:0.0:1788201706.641977:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478134272 chunk: [478134272, 478138368) 478138368, 4096 00020000:00200000:0.0:1788201706.641978:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.641985:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116732, 116732] 00000008:00010000:0.0:1788201706.641988:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.641989:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.641994:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642003:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478134272 478138367) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642011:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642031:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642039:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.642047:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478134272, 478138368) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.642052:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116732 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.642058:0:17463:0:(rw26.c:898:ll_write_end()) pos 478134272, len 4096, copied 4096 00000080:00200000:0.0:1788201706.642061:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000024b98b57[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116732] 00000080:00200000:0.0:1788201706.642063:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bebd8025 fffffc0001029 2:0 ffff93c0c1222460 116732 lru 00000080:00200000:0.0:1788201706.642067:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f1125bda 116732: 1< 2 - - > 2< 478134272 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68317 - -> 00000080:00200000:0.0:1788201706.642068:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000024b98b57 00000080:00200000:0.0:1788201706.642069:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.642070:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.642077:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.642085:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.642086:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.642088:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.642095:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.642096:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.642098:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.642101:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642106:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.642111:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.642112:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.642115:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.642117:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116732, 116732] 00000080:00200000:0.0:1788201706.642124:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.642126:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.642145:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.642147:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.642158:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478138368, count: 4096 00000080:00200000:0.0:1788201706.642164:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478138368, bytes: 4096 00000080:00200000:0.0:1788201706.642166:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.642168:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.642169:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478138368, 478142464) 0 0 00000080:00200000:0.0:1788201706.642171:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.642174:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116733, 116733] 00020000:00200000:0.0:1788201706.642176:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478138368 chunk: [478138368, 478142464) 478142464, 4096 00020000:00200000:0.0:1788201706.642177:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.642184:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116733, 116733] 00000008:00010000:0.0:1788201706.642187:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.642189:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.642194:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642202:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478138368 478142463) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642211:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642218:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642225:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.642238:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478138368, 478142464) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.642243:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116733 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.642248:0:17463:0:(rw26.c:898:ll_write_end()) pos 478138368, len 4096, copied 4096 00000080:00200000:0.0:1788201706.642252:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000014414a8[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116733] 00000080:00200000:0.0:1788201706.642255:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bd5f3825 fffffc0001029 2:0 ffff93c0c12220e0 116733 lru 00000080:00200000:0.0:1788201706.642261:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bc4c2157 116733: 1< 2 - - > 2< 478138368 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68318 - -> 00000080:00200000:0.0:1788201706.642263:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000014414a8 00000080:00200000:0.0:1788201706.642264:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.642266:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.642274:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.642285:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.642287:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.642288:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.642295:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.642297:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.642298:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.642301:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642306:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.642311:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.642312:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.642316:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.642318:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116733, 116733] 00000080:00200000:0.0:1788201706.642325:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.642327:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.642345:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.642347:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.642358:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478142464, count: 4096 00000080:00200000:0.0:1788201706.642364:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478142464, bytes: 4096 00000080:00200000:0.0:1788201706.642366:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.642368:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.642369:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478142464, 478146560) 0 0 00000080:00200000:0.0:1788201706.642371:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.642374:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116734, 116734] 00020000:00200000:0.0:1788201706.642376:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478142464 chunk: [478142464, 478146560) 478146560, 4096 00020000:00200000:0.0:1788201706.642378:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.642384:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116734, 116734] 00000008:00010000:0.0:1788201706.642387:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.642389:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.642394:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642399:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478142464 478146559) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642405:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642409:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642417:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.642422:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478142464, 478146560) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.642424:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116734 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.642428:0:17463:0:(rw26.c:898:ll_write_end()) pos 478142464, len 4096, copied 4096 00000080:00200000:0.0:1788201706.642430:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ecf75b86[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116734] 00000080:00200000:0.0:1788201706.642432:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000780e3624 fffffc0001029 2:0 ffff93c0c1222700 116734 lru 00000080:00200000:0.0:1788201706.642435:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000072872d4c 116734: 1< 2 - - > 2< 478142464 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68319 - -> 00000080:00200000:0.0:1788201706.642437:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ecf75b86 00000080:00200000:0.0:1788201706.642437:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.642439:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.642444:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.642451:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.642453:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.642454:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.642460:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.642462:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.642463:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.642466:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642470:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.642479:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.642480:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.642483:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.642485:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116734, 116734] 00000080:00200000:0.0:1788201706.642491:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.642493:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.642511:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.642513:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.642523:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478146560, count: 4096 00000080:00200000:0.0:1788201706.642529:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478146560, bytes: 4096 00000080:00200000:0.0:1788201706.642531:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.642533:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.642534:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478146560, 478150656) 0 0 00000080:00200000:0.0:1788201706.642535:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.642538:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116735, 116735] 00020000:00200000:0.0:1788201706.642540:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478146560 chunk: [478146560, 478150656) 478150656, 4096 00020000:00200000:0.0:1788201706.642542:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.642548:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116735, 116735] 00000008:00010000:0.0:1788201706.642551:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.642552:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.642557:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642563:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478146560 478150655) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642568:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642577:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642581:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.642585:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478146560, 478150656) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.642588:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116735 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.642591:0:17463:0:(rw26.c:898:ll_write_end()) pos 478146560, len 4096, copied 4096 00000080:00200000:0.0:1788201706.642593:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000af36fac9[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116735] 00000080:00200000:0.0:1788201706.642595:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b18f39e6 fffffc0001029 2:0 ffff93c0c1222000 116735 lru 00000080:00200000:0.0:1788201706.642598:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009de3b74d 116735: 1< 2 - - > 2< 478146560 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68320 - -> 00000080:00200000:0.0:1788201706.642599:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000af36fac9 00000080:00200000:0.0:1788201706.642600:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.642601:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.642606:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000250468a0 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.642614:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.642616:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.642617:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.642622:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.642624:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.642626:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.642628:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642636:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.642641:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.642642:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.642645:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.642646:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116735, 116735] 00000080:00200000:0.0:1788201706.642656:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.642659:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.642690:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.642694:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.642710:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478150656, count: 4096 00000080:00200000:0.0:1788201706.642721:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478150656, bytes: 4096 00000080:00200000:0.0:1788201706.642724:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.642728:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.642729:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478150656, 478154752) 0 0 00000080:00200000:0.0:1788201706.642732:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.642737:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116736, 116736] 00020000:00200000:0.0:1788201706.642740:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478150656 chunk: [478150656, 478154752) 478154752, 4096 00020000:00200000:0.0:1788201706.642743:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.642755:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116736, 116736] 00000008:00010000:0.0:1788201706.642761:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.642763:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.642770:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642780:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478150656 478154751) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 278/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642794:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642802:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642809:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.642817:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478150656, 478154752) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.642822:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116736 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.642828:0:17463:0:(rw26.c:898:ll_write_end()) pos 478150656, len 4096, copied 4096 00000080:00200000:0.0:1788201706.642849:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e232cbc2[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116736] 00000080:00200000:0.0:1788201706.642852:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002f245154 fffffc0001029 2:0 ffff93c0c1330c40 116736 lru 00000080:00200000:0.0:1788201706.642858:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001e1bae59 116736: 1< 2 - - > 2< 478150656 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 17039360 - | + - + - > 5< + - + - | 0 - | 68321 - -> 00000080:00200000:0.0:1788201706.642861:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e232cbc2 00000080:00200000:0.0:1788201706.642862:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.642864:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.642875:0:17463:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.642886:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.642890:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.642892:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.642901:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.642904:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.642907:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.642911:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.642919:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.642928:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.642930:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.642935:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.642938:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116736, 116736] 00000080:00200000:0.0:1788201706.642949:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.642952:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.642981:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.642985:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.643001:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478154752, count: 4096 00000080:00200000:0.0:1788201706.643012:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478154752, bytes: 4096 00000080:00200000:0.0:1788201706.643026:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.643031:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.643032:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478154752, 478158848) 0 0 00000080:00200000:0.0:1788201706.643035:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.643040:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116737, 116737] 00020000:00200000:0.0:1788201706.643043:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478154752 chunk: [478154752, 478158848) 478158848, 4096 00020000:00200000:0.0:1788201706.643045:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.643058:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116737, 116737] 00000008:00010000:0.0:1788201706.643064:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.643066:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.643075:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643089:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478154752 478158847) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643098:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643105:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643113:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.643121:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478154752, 478158848) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.643127:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116737 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.643132:0:17463:0:(rw26.c:898:ll_write_end()) pos 478154752, len 4096, copied 4096 00000080:00200000:0.0:1788201706.643136:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008b1e922a[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116737] 00000080:00200000:0.0:1788201706.643139:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e33622a8 fffffc0001029 2:0 ffff93c0c1330540 116737 lru 00000080:00200000:0.0:1788201706.643146:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006f8e87bb 116737: 1< 2 - - > 2< 478154752 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68322 - -> 00000080:00200000:0.0:1788201706.643148:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008b1e922a 00000080:00200000:0.0:1788201706.643149:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.643151:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.643160:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.643172:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.643176:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.643178:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.643189:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.643192:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.643195:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.643199:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643207:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.643215:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.643218:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.643222:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.643225:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116737, 116737] 00000080:00200000:0.0:1788201706.643235:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.643238:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.643267:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.643270:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.643285:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478158848, count: 4096 00000080:00200000:0.0:1788201706.643294:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478158848, bytes: 4096 00000080:00200000:0.0:1788201706.643297:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.643300:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.643302:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478158848, 478162944) 0 0 00000080:00200000:0.0:1788201706.643304:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.643309:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116738, 116738] 00020000:00200000:0.0:1788201706.643313:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478158848 chunk: [478158848, 478162944) 478162944, 4096 00020000:00200000:0.0:1788201706.643315:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.643326:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116738, 116738] 00000008:00010000:0.0:1788201706.643331:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.643333:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.643341:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643349:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478158848 478162943) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643357:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643364:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643370:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.643378:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478158848, 478162944) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.643383:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116738 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.643388:0:17463:0:(rw26.c:898:ll_write_end()) pos 478158848, len 4096, copied 4096 00000080:00200000:0.0:1788201706.643392:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a8b0d651[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116738] 00000080:00200000:0.0:1788201706.643395:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e8fc1790 fffffc0001029 2:0 ffff93c0c1330460 116738 lru 00000080:00200000:0.0:1788201706.643401:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000000b37413 116738: 1< 2 - - > 2< 478158848 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68323 - -> 00000080:00200000:0.0:1788201706.643404:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a8b0d651 00000080:00200000:0.0:1788201706.643405:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.643407:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.643415:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.643427:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.643430:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.643433:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.643442:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.643445:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.643447:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.643451:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643460:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.643469:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.643471:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.643476:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.643479:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116738, 116738] 00000080:00200000:0.0:1788201706.643491:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.643494:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.643525:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.643529:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.643545:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478162944, count: 4096 00000080:00200000:0.0:1788201706.643556:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478162944, bytes: 4096 00000080:00200000:0.0:1788201706.643559:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.643563:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.643564:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478162944, 478167040) 0 0 00000080:00200000:0.0:1788201706.643567:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.643572:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116739, 116739] 00020000:00200000:0.0:1788201706.643575:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478162944 chunk: [478162944, 478167040) 478167040, 4096 00020000:00200000:0.0:1788201706.643577:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.643588:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116739, 116739] 00000008:00010000:0.0:1788201706.643593:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.643596:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.643604:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643612:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478162944 478167039) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643620:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643628:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643635:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.643643:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478162944, 478167040) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.643648:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116739 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.643654:0:17463:0:(rw26.c:898:ll_write_end()) pos 478162944, len 4096, copied 4096 00000080:00200000:0.0:1788201706.643656:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ed532db6[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116739] 00000080:00200000:0.0:1788201706.643658:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e0d136a4 fffffc0001029 2:0 ffff93c0c13300e0 116739 lru 00000080:00200000:0.0:1788201706.643662:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c1e8f72d 116739: 1< 2 - - > 2< 478162944 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68324 - -> 00000080:00200000:0.0:1788201706.643664:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ed532db6 00000080:00200000:0.0:1788201706.643665:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.643666:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.643672:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.643684:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.643685:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.643687:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.643693:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.643695:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.643697:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.643699:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643704:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.643710:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.643711:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.643713:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.643715:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116739, 116739] 00000080:00200000:0.0:1788201706.643722:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.643724:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.643743:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.643745:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.643755:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478167040, count: 4096 00000080:00200000:0.0:1788201706.643761:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478167040, bytes: 4096 00000080:00200000:0.0:1788201706.643763:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.643766:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.643767:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478167040, 478171136) 0 0 00000080:00200000:0.0:1788201706.643768:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.643772:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116740, 116740] 00020000:00200000:0.0:1788201706.643774:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478167040 chunk: [478167040, 478171136) 478171136, 4096 00020000:00200000:0.0:1788201706.643775:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.643782:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116740, 116740] 00000008:00010000:0.0:1788201706.643785:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.643787:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.643792:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643798:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478167040 478171135) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643803:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643808:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643812:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.643816:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478167040, 478171136) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.643819:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116740 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.643823:0:17463:0:(rw26.c:898:ll_write_end()) pos 478167040, len 4096, copied 4096 00000080:00200000:0.0:1788201706.643826:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000005346140[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116740] 00000080:00200000:0.0:1788201706.643827:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f527ec35 fffffc0001029 2:0 ffff93c0c1330700 116740 lru 00000080:00200000:0.0:1788201706.643845:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000053120178 116740: 1< 2 - - > 2< 478167040 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68325 - -> 00000080:00200000:0.0:1788201706.643847:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000005346140 00000080:00200000:0.0:1788201706.643848:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.643849:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.643856:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.643864:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.643866:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.643867:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.643873:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.643875:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.643876:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.643879:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643884:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.643888:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.643890:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.643892:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.643894:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116740, 116740] 00000080:00200000:0.0:1788201706.643901:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.643903:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.643921:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.643923:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.643933:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478171136, count: 4096 00000080:00200000:0.0:1788201706.643939:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478171136, bytes: 4096 00000080:00200000:0.0:1788201706.643941:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.643943:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.643944:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478171136, 478175232) 0 0 00000080:00200000:0.0:1788201706.643946:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.643949:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116741, 116741] 00020000:00200000:0.0:1788201706.643951:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478171136 chunk: [478171136, 478175232) 478175232, 4096 00020000:00200000:0.0:1788201706.643952:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.643958:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116741, 116741] 00000008:00010000:0.0:1788201706.643962:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.643963:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.643968:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643974:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478171136 478175231) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643979:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643984:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.643987:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.643994:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478171136, 478175232) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.643999:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116741 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.644003:0:17463:0:(rw26.c:898:ll_write_end()) pos 478171136, len 4096, copied 4096 00000080:00200000:0.0:1788201706.644007:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000eea4ba2b[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116741] 00000080:00200000:0.0:1788201706.644010:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000595371f4 fffffc0001029 2:0 ffff93c0c1330000 116741 lru 00000080:00200000:0.0:1788201706.644027:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f2e7612e 116741: 1< 2 - - > 2< 478171136 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68326 - -> 00000080:00200000:0.0:1788201706.644032:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000eea4ba2b 00000080:00200000:0.0:1788201706.644033:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.644035:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.644044:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.644056:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.644059:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.644062:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.644071:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.644074:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.644076:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.644080:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644088:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.644096:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.644099:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.644103:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.644106:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116741, 116741] 00000080:00200000:0.0:1788201706.644116:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.644120:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.644149:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.644152:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.644168:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478175232, count: 4096 00000080:00200000:0.0:1788201706.644179:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478175232, bytes: 4096 00000080:00200000:0.0:1788201706.644182:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.644186:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.644188:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478175232, 478179328) 0 0 00000080:00200000:0.0:1788201706.644190:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.644195:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116742, 116742] 00020000:00200000:0.0:1788201706.644198:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478175232 chunk: [478175232, 478179328) 478179328, 4096 00020000:00200000:0.0:1788201706.644200:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.644212:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116742, 116742] 00000008:00010000:0.0:1788201706.644217:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.644220:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.644227:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644237:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478175232 478179327) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644245:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644252:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644260:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.644267:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478175232, 478179328) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.644272:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116742 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.644277:0:17463:0:(rw26.c:898:ll_write_end()) pos 478175232, len 4096, copied 4096 00000080:00200000:0.0:1788201706.644281:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000378ff522[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116742] 00000080:00200000:0.0:1788201706.644285:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000029202db fffffc0001029 2:0 ffff93c0c1330a80 116742 lru 00000080:00200000:0.0:1788201706.644289:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005c251603 116742: 1< 2 - - > 2< 478175232 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68327 - -> 00000080:00200000:0.0:1788201706.644291:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000378ff522 00000080:00200000:0.0:1788201706.644292:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.644293:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.644300:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.644308:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.644310:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.644311:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.644318:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.644319:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.644321:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.644323:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644328:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.644334:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.644335:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.644337:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.644339:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116742, 116742] 00000080:00200000:0.0:1788201706.644346:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.644348:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.644367:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.644369:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.644380:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478179328, count: 4096 00000080:00200000:0.0:1788201706.644386:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478179328, bytes: 4096 00000080:00200000:0.0:1788201706.644388:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.644390:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.644391:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478179328, 478183424) 0 0 00000080:00200000:0.0:1788201706.644393:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.644397:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116743, 116743] 00020000:00200000:0.0:1788201706.644398:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478179328 chunk: [478179328, 478183424) 478183424, 4096 00020000:00200000:0.0:1788201706.644400:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.644406:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116743, 116743] 00000008:00010000:0.0:1788201706.644410:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.644411:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.644416:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644422:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478179328 478183423) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644427:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644432:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644436:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.644440:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478179328, 478183424) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.644443:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116743 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.644447:0:17463:0:(rw26.c:898:ll_write_end()) pos 478179328, len 4096, copied 4096 00000080:00200000:0.0:1788201706.644450:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007fc513aa[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116743] 00000080:00200000:0.0:1788201706.644451:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000018b7e46d fffffc0001029 2:0 ffff93c0c1330380 116743 lru 00000080:00200000:0.0:1788201706.644455:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ab7a92d1 116743: 1< 2 - - > 2< 478179328 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68328 - -> 00000080:00200000:0.0:1788201706.644457:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007fc513aa 00000080:00200000:0.0:1788201706.644457:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.644459:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.644464:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.644471:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.644473:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.644474:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.644480:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.644482:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.644484:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.644486:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644491:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.644496:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.644497:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.644499:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.644501:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116743, 116743] 00000080:00200000:0.0:1788201706.644508:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.644510:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.644528:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.644530:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.644540:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478183424, count: 4096 00000080:00200000:0.0:1788201706.644545:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478183424, bytes: 4096 00000080:00200000:0.0:1788201706.644547:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.644549:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.644550:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478183424, 478187520) 0 0 00000080:00200000:0.0:1788201706.644552:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.644555:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116744, 116744] 00020000:00200000:0.0:1788201706.644557:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478183424 chunk: [478183424, 478187520) 478187520, 4096 00020000:00200000:0.0:1788201706.644559:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.644565:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116744, 116744] 00000008:00010000:0.0:1788201706.644568:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.644569:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.644574:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644580:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478183424 478187519) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644585:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644589:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644593:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.644597:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478183424, 478187520) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.644601:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116744 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.644605:0:17463:0:(rw26.c:898:ll_write_end()) pos 478183424, len 4096, copied 4096 00000080:00200000:0.0:1788201706.644607:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000098f24f91[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116744] 00000080:00200000:0.0:1788201706.644609:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f61211c3 fffffc0001029 2:0 ffff93c0c13308c0 116744 lru 00000080:00200000:0.0:1788201706.644612:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e47e1b34 116744: 1< 2 - - > 2< 478183424 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68329 - -> 00000080:00200000:0.0:1788201706.644614:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000098f24f91 00000080:00200000:0.0:1788201706.644615:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.644616:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.644621:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.644628:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.644630:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.644631:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.644637:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.644639:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.644640:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.644643:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644647:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.644652:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.644653:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.644656:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.644658:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116744, 116744] 00000080:00200000:0.0:1788201706.644664:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.644666:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.644684:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.644686:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.644696:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478187520, count: 4096 00000080:00200000:0.0:1788201706.644702:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478187520, bytes: 4096 00000080:00200000:0.0:1788201706.644704:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.644706:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.644707:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478187520, 478191616) 0 0 00000080:00200000:0.0:1788201706.644709:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.644712:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116745, 116745] 00020000:00200000:0.0:1788201706.644713:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478187520 chunk: [478187520, 478191616) 478191616, 4096 00020000:00200000:0.0:1788201706.644715:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.644721:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116745, 116745] 00000008:00010000:0.0:1788201706.644724:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.644726:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.644731:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644736:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478187520 478191615) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644742:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644746:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644750:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.644758:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478187520, 478191616) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.644761:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116745 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.644765:0:17463:0:(rw26.c:898:ll_write_end()) pos 478187520, len 4096, copied 4096 00000080:00200000:0.0:1788201706.644767:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bb473de2[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116745] 00000080:00200000:0.0:1788201706.644769:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000093b1a872 fffffc0001029 2:0 ffff93c0c1330ee0 116745 lru 00000080:00200000:0.0:1788201706.644773:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002079dad3 116745: 1< 2 - - > 2< 478187520 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68330 - -> 00000080:00200000:0.0:1788201706.644774:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bb473de2 00000080:00200000:0.0:1788201706.644775:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.644776:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.644781:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.644788:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.644789:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.644790:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.644796:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.644798:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.644799:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.644801:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644806:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.644810:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.644812:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.644815:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.644817:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116745, 116745] 00000080:00200000:0.0:1788201706.644823:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.644825:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.644863:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.644865:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.644877:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478191616, count: 4096 00000080:00200000:0.0:1788201706.644883:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478191616, bytes: 4096 00000080:00200000:0.0:1788201706.644885:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.644887:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.644888:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478191616, 478195712) 0 0 00000080:00200000:0.0:1788201706.644890:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.644893:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116746, 116746] 00020000:00200000:0.0:1788201706.644895:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478191616 chunk: [478191616, 478195712) 478195712, 4096 00020000:00200000:0.0:1788201706.644896:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.644903:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116746, 116746] 00000008:00010000:0.0:1788201706.644906:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.644907:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.644912:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644918:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478191616 478195711) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644923:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644927:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644935:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.644939:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478191616, 478195712) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.644942:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116746 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.644946:0:17463:0:(rw26.c:898:ll_write_end()) pos 478191616, len 4096, copied 4096 00000080:00200000:0.0:1788201706.644948:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006fc595a7[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116746] 00000080:00200000:0.0:1788201706.644950:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000be152239 fffffc0001029 2:0 ffff93c0c13307e0 116746 lru 00000080:00200000:0.0:1788201706.644953:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a5ff96b3 116746: 1< 2 - - > 2< 478191616 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68331 - -> 00000080:00200000:0.0:1788201706.644955:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006fc595a7 00000080:00200000:0.0:1788201706.644955:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.644956:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.644962:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.644969:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.644971:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.644972:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.644978:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.644980:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.644981:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.644984:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.644989:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.644997:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.644999:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.645001:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.645003:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116746, 116746] 00000080:00200000:0.0:1788201706.645010:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.645012:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.645045:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.645047:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.645058:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478195712, count: 4096 00000080:00200000:0.0:1788201706.645064:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478195712, bytes: 4096 00000080:00200000:0.0:1788201706.645066:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.645068:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.645069:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478195712, 478199808) 0 0 00000080:00200000:0.0:1788201706.645071:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.645074:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116747, 116747] 00020000:00200000:0.0:1788201706.645076:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478195712 chunk: [478195712, 478199808) 478199808, 4096 00020000:00200000:0.0:1788201706.645077:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.645084:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116747, 116747] 00000008:00010000:0.0:1788201706.645088:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.645090:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.645097:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645106:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478195712 478199807) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645115:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645127:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645135:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.645142:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478195712, 478199808) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.645148:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116747 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.645153:0:17463:0:(rw26.c:898:ll_write_end()) pos 478195712, len 4096, copied 4096 00000080:00200000:0.0:1788201706.645157:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005bfb4950[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116747] 00000080:00200000:0.0:1788201706.645159:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000099e909fb fffffc0001029 2:0 ffff93c0c13302a0 116747 lru 00000080:00200000:0.0:1788201706.645162:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003a21b658 116747: 1< 2 - - > 2< 478195712 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68332 - -> 00000080:00200000:0.0:1788201706.645164:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005bfb4950 00000080:00200000:0.0:1788201706.645165:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.645166:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.645172:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.645180:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.645182:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.645183:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.645190:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.645192:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.645193:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.645197:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645209:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.645217:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.645219:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.645223:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.645226:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116747, 116747] 00000080:00200000:0.0:1788201706.645238:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.645241:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.645269:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.645272:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.645287:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478199808, count: 4096 00000080:00200000:0.0:1788201706.645297:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478199808, bytes: 4096 00000080:00200000:0.0:1788201706.645299:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.645302:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.645304:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478199808, 478203904) 0 0 00000080:00200000:0.0:1788201706.645306:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.645311:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116748, 116748] 00020000:00200000:0.0:1788201706.645314:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478199808 chunk: [478199808, 478203904) 478203904, 4096 00020000:00200000:0.0:1788201706.645316:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.645326:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116748, 116748] 00000008:00010000:0.0:1788201706.645331:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.645333:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.645340:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645348:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478199808 478203903) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645361:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645368:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645374:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.645380:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478199808, 478203904) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.645385:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116748 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.645390:0:17463:0:(rw26.c:898:ll_write_end()) pos 478199808, len 4096, copied 4096 00000080:00200000:0.0:1788201706.645394:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000aa930f38[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116748] 00000080:00200000:0.0:1788201706.645397:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000397a8ea8 fffffc0001029 2:0 ffff93c0c1331b60 116748 lru 00000080:00200000:0.0:1788201706.645402:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000072ccd996 116748: 1< 2 - - > 2< 478199808 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68333 - -> 00000080:00200000:0.0:1788201706.645404:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000aa930f38 00000080:00200000:0.0:1788201706.645405:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.645407:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.645414:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.645425:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.645428:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.645430:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.645439:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.645442:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.645445:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.645449:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645456:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.645464:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.645466:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.645470:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.645473:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116748, 116748] 00000080:00200000:0.0:1788201706.645484:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.645486:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.645511:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.645515:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.645530:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478203904, count: 4096 00000080:00200000:0.0:1788201706.645541:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478203904, bytes: 4096 00000080:00200000:0.0:1788201706.645544:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.645548:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.645549:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478203904, 478208000) 0 0 00000080:00200000:0.0:1788201706.645552:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.645557:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116749, 116749] 00020000:00200000:0.0:1788201706.645560:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478203904 chunk: [478203904, 478208000) 478208000, 4096 00020000:00200000:0.0:1788201706.645562:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.645575:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116749, 116749] 00000008:00010000:0.0:1788201706.645580:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.645582:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.645590:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645604:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478203904 478207999) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645614:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645621:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645628:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.645636:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478203904, 478208000) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.645641:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116749 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.645647:0:17463:0:(rw26.c:898:ll_write_end()) pos 478203904, len 4096, copied 4096 00000080:00200000:0.0:1788201706.645651:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fa9eeb2f[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116749] 00000080:00200000:0.0:1788201706.645654:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fad10b88 fffffc0001029 2:0 ffff93c0c1331c40 116749 lru 00000080:00200000:0.0:1788201706.645660:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000945ea41b 116749: 1< 2 - - > 2< 478203904 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68334 - -> 00000080:00200000:0.0:1788201706.645662:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fa9eeb2f 00000080:00200000:0.0:1788201706.645663:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.645665:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.645674:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.645686:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.645689:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.645692:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.645700:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.645703:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.645706:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.645710:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645718:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.645726:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.645729:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.645733:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.645736:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116749, 116749] 00000080:00200000:0.0:1788201706.645746:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.645750:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.645775:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.645779:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.645794:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478208000, count: 4096 00000080:00200000:0.0:1788201706.645805:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478208000, bytes: 4096 00000080:00200000:0.0:1788201706.645808:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.645811:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.645813:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478208000, 478212096) 0 0 00000080:00200000:0.0:1788201706.645815:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.645820:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116750, 116750] 00020000:00200000:0.0:1788201706.645823:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478208000 chunk: [478208000, 478212096) 478212096, 4096 00020000:00200000:0.0:1788201706.645825:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.645855:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116750, 116750] 00000008:00010000:0.0:1788201706.645860:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.645862:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.645870:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645878:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478208000 478212095) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645886:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645892:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645899:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.645905:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478208000, 478212096) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.645910:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116750 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.645915:0:17463:0:(rw26.c:898:ll_write_end()) pos 478208000, len 4096, copied 4096 00000080:00200000:0.0:1788201706.645919:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000f98f7a0[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116750] 00000080:00200000:0.0:1788201706.645922:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000116b59d fffffc0001029 2:0 ffff93c0c1331540 116750 lru 00000080:00200000:0.0:1788201706.645928:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000074c0b290 116750: 1< 2 - - > 2< 478208000 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68335 - -> 00000080:00200000:0.0:1788201706.645930:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000f98f7a0 00000080:00200000:0.0:1788201706.645931:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.645933:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.645941:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.645954:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.645957:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.645960:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.645969:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.645972:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.645974:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.645978:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.645986:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.645993:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.645994:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.645997:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.645999:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116750, 116750] 00000080:00200000:0.0:1788201706.646007:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.646009:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.646042:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.646045:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.646056:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478212096, count: 4096 00000080:00200000:0.0:1788201706.646062:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478212096, bytes: 4096 00000080:00200000:0.0:1788201706.646064:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.646066:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.646068:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478212096, 478216192) 0 0 00000080:00200000:0.0:1788201706.646069:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.646072:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116751, 116751] 00020000:00200000:0.0:1788201706.646075:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478212096 chunk: [478212096, 478216192) 478216192, 4096 00020000:00200000:0.0:1788201706.646076:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.646083:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116751, 116751] 00000008:00010000:0.0:1788201706.646087:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.646088:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.646094:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646099:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478212096 478216191) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646105:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646109:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646114:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.646118:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478212096, 478216192) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.646121:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116751 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.646124:0:17463:0:(rw26.c:898:ll_write_end()) pos 478212096, len 4096, copied 4096 00000080:00200000:0.0:1788201706.646127:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005d934869[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116751] 00000080:00200000:0.0:1788201706.646129:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000029e0a558 fffffc0001029 2:0 ffff93c0c1331460 116751 lru 00000080:00200000:0.0:1788201706.646132:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006774d5ad 116751: 1< 2 - - > 2< 478212096 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68336 - -> 00000080:00200000:0.0:1788201706.646134:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005d934869 00000080:00200000:0.0:1788201706.646134:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.646136:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.646143:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.646160:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.646163:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.646165:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.646174:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.646176:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.646178:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.646182:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646189:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.646197:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.646199:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.646203:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.646206:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116751, 116751] 00000080:00200000:0.0:1788201706.646216:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.646219:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.646246:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.646250:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.646264:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478216192, count: 4096 00000080:00200000:0.0:1788201706.646275:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478216192, bytes: 4096 00000080:00200000:0.0:1788201706.646278:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.646281:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.646283:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478216192, 478220288) 0 0 00000080:00200000:0.0:1788201706.646286:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.646290:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116752, 116752] 00020000:00200000:0.0:1788201706.646294:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478216192 chunk: [478216192, 478220288) 478220288, 4096 00020000:00200000:0.0:1788201706.646296:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.646305:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116752, 116752] 00000008:00010000:0.0:1788201706.646308:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.646310:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.646315:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646321:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478216192 478220287) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646326:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646331:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646335:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.646340:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478216192, 478220288) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.646343:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116752 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.646347:0:17463:0:(rw26.c:898:ll_write_end()) pos 478216192, len 4096, copied 4096 00000080:00200000:0.0:1788201706.646349:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004339f42f[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116752] 00000080:00200000:0.0:1788201706.646351:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d0033e22 fffffc0001029 2:0 ffff93c0c13310e0 116752 lru 00000080:00200000:0.0:1788201706.646355:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b82bd14c 116752: 1< 2 - - > 2< 478216192 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68337 - -> 00000080:00200000:0.0:1788201706.646356:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004339f42f 00000080:00200000:0.0:1788201706.646357:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.646359:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.646364:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.646372:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.646374:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.646375:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.646382:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.646384:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.646385:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.646388:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646392:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.646397:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.646398:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.646401:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.646403:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116752, 116752] 00000080:00200000:0.0:1788201706.646410:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.646412:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.646430:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.646432:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.646442:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478220288, count: 4096 00000080:00200000:0.0:1788201706.646448:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478220288, bytes: 4096 00000080:00200000:0.0:1788201706.646450:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.646452:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.646453:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478220288, 478224384) 0 0 00000080:00200000:0.0:1788201706.646455:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.646458:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116753, 116753] 00020000:00200000:0.0:1788201706.646460:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478220288 chunk: [478220288, 478224384) 478224384, 4096 00020000:00200000:0.0:1788201706.646462:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.646468:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116753, 116753] 00000008:00010000:0.0:1788201706.646471:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.646472:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.646477:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646483:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478220288 478224383) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646488:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646493:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646497:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.646501:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478220288, 478224384) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.646505:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116753 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.646508:0:17463:0:(rw26.c:898:ll_write_end()) pos 478220288, len 4096, copied 4096 00000080:00200000:0.0:1788201706.646510:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005b39d781[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116753] 00000080:00200000:0.0:1788201706.646512:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000583848d2 fffffc0001029 2:0 ffff93c0c1331700 116753 lru 00000080:00200000:0.0:1788201706.646516:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ae2bb7c5 116753: 1< 2 - - > 2< 478220288 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68338 - -> 00000080:00200000:0.0:1788201706.646519:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005b39d781 00000080:00200000:0.0:1788201706.646519:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.646521:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.646526:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.646533:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.646535:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.646536:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.646542:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.646544:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.646545:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.646548:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646553:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.646558:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.646559:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.646561:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.646563:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116753, 116753] 00000080:00200000:0.0:1788201706.646570:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.646571:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.646589:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.646591:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.646601:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478224384, count: 4096 00000080:00200000:0.0:1788201706.646607:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478224384, bytes: 4096 00000080:00200000:0.0:1788201706.646609:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.646611:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.646612:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478224384, 478228480) 0 0 00000080:00200000:0.0:1788201706.646614:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.646617:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116754, 116754] 00020000:00200000:0.0:1788201706.646618:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478224384 chunk: [478224384, 478228480) 478228480, 4096 00020000:00200000:0.0:1788201706.646620:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.646626:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116754, 116754] 00000008:00010000:0.0:1788201706.646629:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.646631:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.646636:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646641:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478224384 478228479) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646647:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646651:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646655:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.646659:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478224384, 478228480) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.646662:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116754 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.646666:0:17463:0:(rw26.c:898:ll_write_end()) pos 478224384, len 4096, copied 4096 00000080:00200000:0.0:1788201706.646668:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009e5b2c4b[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116754] 00000080:00200000:0.0:1788201706.646670:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000dff60f14 fffffc0001029 2:0 ffff93c0c1331000 116754 lru 00000080:00200000:0.0:1788201706.646673:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f9948b3f 116754: 1< 2 - - > 2< 478224384 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68339 - -> 00000080:00200000:0.0:1788201706.646675:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009e5b2c4b 00000080:00200000:0.0:1788201706.646676:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.646677:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.646682:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.646689:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.646690:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.646692:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.646697:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.646699:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.646701:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.646703:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646707:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.646712:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.646713:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.646716:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.646718:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116754, 116754] 00000080:00200000:0.0:1788201706.646724:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.646726:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.646744:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.646746:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.646756:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478228480, count: 4096 00000080:00200000:0.0:1788201706.646762:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478228480, bytes: 4096 00000080:00200000:0.0:1788201706.646764:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.646766:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.646767:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478228480, 478232576) 0 0 00000080:00200000:0.0:1788201706.646768:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.646771:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116755, 116755] 00020000:00200000:0.0:1788201706.646773:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478228480 chunk: [478228480, 478232576) 478232576, 4096 00020000:00200000:0.0:1788201706.646775:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.646781:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116755, 116755] 00000008:00010000:0.0:1788201706.646784:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.646785:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.646790:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646796:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478228480 478232575) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646801:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646806:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646810:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.646814:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478228480, 478232576) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.646817:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116755 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.646821:0:17463:0:(rw26.c:898:ll_write_end()) pos 478228480, len 4096, copied 4096 00000080:00200000:0.0:1788201706.646823:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001b1c26d5[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116755] 00000080:00200000:0.0:1788201706.646825:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006d895c30 fffffc0001029 2:0 ffff93c0c1331a80 116755 lru 00000080:00200000:0.0:1788201706.646842:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000078afd92b 116755: 1< 2 - - > 2< 478228480 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68340 - -> 00000080:00200000:0.0:1788201706.646844:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001b1c26d5 00000080:00200000:0.0:1788201706.646844:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.646846:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.646852:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.646860:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.646862:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.646863:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.646869:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.646871:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.646872:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.646875:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646880:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.646885:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.646886:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.646888:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.646890:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116755, 116755] 00000080:00200000:0.0:1788201706.646896:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.646898:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.646916:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.646919:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.646929:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478232576, count: 4096 00000080:00200000:0.0:1788201706.646935:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478232576, bytes: 4096 00000080:00200000:0.0:1788201706.646936:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.646938:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.646939:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478232576, 478236672) 0 0 00000080:00200000:0.0:1788201706.646941:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.646944:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116756, 116756] 00020000:00200000:0.0:1788201706.646946:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478232576 chunk: [478232576, 478236672) 478236672, 4096 00020000:00200000:0.0:1788201706.646948:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.646954:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116756, 116756] 00000008:00010000:0.0:1788201706.646957:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.646958:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.646963:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646969:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478232576 478236671) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646974:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646979:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.646983:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.646987:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478232576, 478236672) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.646990:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116756 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.646994:0:17463:0:(rw26.c:898:ll_write_end()) pos 478232576, len 4096, copied 4096 00000080:00200000:0.0:1788201706.646996:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004be8b7aa[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116756] 00000080:00200000:0.0:1788201706.646998:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000636b9d2e fffffc0001029 2:0 ffff93c0c1331380 116756 lru 00000080:00200000:0.0:1788201706.647002:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cd2ff6b9 116756: 1< 2 - - > 2< 478232576 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68341 - -> 00000080:00200000:0.0:1788201706.647003:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004be8b7aa 00000080:00200000:0.0:1788201706.647004:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.647005:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.647011:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.647030:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.647032:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.647034:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.647040:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.647042:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.647043:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.647046:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647051:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.647055:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.647057:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.647059:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.647061:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116756, 116756] 00000080:00200000:0.0:1788201706.647068:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.647070:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.647088:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.647090:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.647100:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478236672, count: 4096 00000080:00200000:0.0:1788201706.647106:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478236672, bytes: 4096 00000080:00200000:0.0:1788201706.647108:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.647110:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.647111:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478236672, 478240768) 0 0 00000080:00200000:0.0:1788201706.647112:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.647116:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116757, 116757] 00020000:00200000:0.0:1788201706.647117:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478236672 chunk: [478236672, 478240768) 478240768, 4096 00020000:00200000:0.0:1788201706.647119:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.647125:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116757, 116757] 00000008:00010000:0.0:1788201706.647128:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.647129:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.647134:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647140:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478236672 478240767) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647146:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647150:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647154:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.647162:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478236672, 478240768) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.647165:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116757 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.647169:0:17463:0:(rw26.c:898:ll_write_end()) pos 478236672, len 4096, copied 4096 00000080:00200000:0.0:1788201706.647171:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bc2a35da[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116757] 00000080:00200000:0.0:1788201706.647174:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000726b919a fffffc0001029 2:0 ffff93c0c13318c0 116757 lru 00000080:00200000:0.0:1788201706.647178:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000016ff3310 116757: 1< 2 - - > 2< 478236672 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68342 - -> 00000080:00200000:0.0:1788201706.647179:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bc2a35da 00000080:00200000:0.0:1788201706.647180:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.647181:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.647186:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.647193:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.647195:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.647196:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.647202:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.647203:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.647205:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.647207:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647212:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.647216:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.647218:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.647221:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.647223:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116757, 116757] 00000080:00200000:0.0:1788201706.647229:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.647231:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.647248:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.647250:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.647261:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478240768, count: 4096 00000080:00200000:0.0:1788201706.647266:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478240768, bytes: 4096 00000080:00200000:0.0:1788201706.647268:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.647270:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.647271:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478240768, 478244864) 0 0 00000080:00200000:0.0:1788201706.647273:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.647276:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116758, 116758] 00020000:00200000:0.0:1788201706.647278:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478240768 chunk: [478240768, 478244864) 478244864, 4096 00020000:00200000:0.0:1788201706.647279:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.647285:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116758, 116758] 00000008:00010000:0.0:1788201706.647288:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.647290:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.647295:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647300:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478240768 478244863) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647306:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647310:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647318:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.647322:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478240768, 478244864) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.647325:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116758 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.647329:0:17463:0:(rw26.c:898:ll_write_end()) pos 478240768, len 4096, copied 4096 00000080:00200000:0.0:1788201706.647331:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000452636c7[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116758] 00000080:00200000:0.0:1788201706.647332:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007f3767e1 fffffc0001029 2:0 ffff93c0c1331ee0 116758 lru 00000080:00200000:0.0:1788201706.647336:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003be94050 116758: 1< 2 - - > 2< 478240768 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68343 - -> 00000080:00200000:0.0:1788201706.647337:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000452636c7 00000080:00200000:0.0:1788201706.647338:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.647339:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.647344:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.647351:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.647353:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.647355:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.647361:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.647362:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.647364:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.647366:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647371:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.647379:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.647380:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.647383:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.647385:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116758, 116758] 00000080:00200000:0.0:1788201706.647391:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.647393:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.647411:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.647413:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.647423:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478244864, count: 4096 00000080:00200000:0.0:1788201706.647429:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478244864, bytes: 4096 00000080:00200000:0.0:1788201706.647431:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.647433:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.647434:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478244864, 478248960) 0 0 00000080:00200000:0.0:1788201706.647435:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.647438:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116759, 116759] 00020000:00200000:0.0:1788201706.647440:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478244864 chunk: [478244864, 478248960) 478248960, 4096 00020000:00200000:0.0:1788201706.647442:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.647448:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116759, 116759] 00000008:00010000:0.0:1788201706.647451:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.647452:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.647457:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647463:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478244864 478248959) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647468:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647476:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647480:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.647484:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478244864, 478248960) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.647487:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116759 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.647491:0:17463:0:(rw26.c:898:ll_write_end()) pos 478244864, len 4096, copied 4096 00000080:00200000:0.0:1788201706.647493:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006aa5947c[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116759] 00000080:00200000:0.0:1788201706.647496:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008af29917 fffffc0001029 2:0 ffff93c0c13317e0 116759 lru 00000080:00200000:0.0:1788201706.647501:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000637ee19c 116759: 1< 2 - - > 2< 478244864 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68344 - -> 00000080:00200000:0.0:1788201706.647503:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006aa5947c 00000080:00200000:0.0:1788201706.647504:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.647506:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.647513:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.647525:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.647528:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.647530:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.647539:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.647542:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.647545:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.647549:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647561:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.647569:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.647572:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.647576:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.647578:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116759, 116759] 00000080:00200000:0.0:1788201706.647586:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.647588:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.647607:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.647609:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.647620:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478248960, count: 4096 00000080:00200000:0.0:1788201706.647626:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478248960, bytes: 4096 00000080:00200000:0.0:1788201706.647628:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.647630:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.647631:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478248960, 478253056) 0 0 00000080:00200000:0.0:1788201706.647632:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.647636:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116760, 116760] 00020000:00200000:0.0:1788201706.647638:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478248960 chunk: [478248960, 478253056) 478253056, 4096 00020000:00200000:0.0:1788201706.647639:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.647645:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116760, 116760] 00000008:00010000:0.0:1788201706.647649:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.647650:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.647655:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647661:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478248960 478253055) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647670:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647674:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647678:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.647682:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478248960, 478253056) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.647685:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116760 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.647689:0:17463:0:(rw26.c:898:ll_write_end()) pos 478248960, len 4096, copied 4096 00000080:00200000:0.0:1788201706.647691:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c3794b27[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116760] 00000080:00200000:0.0:1788201706.647693:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000032c850eb fffffc0001029 2:0 ffff93c0c13312a0 116760 lru 00000080:00200000:0.0:1788201706.647696:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f6ce4fc4 116760: 1< 2 - - > 2< 478248960 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68345 - -> 00000080:00200000:0.0:1788201706.647698:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c3794b27 00000080:00200000:0.0:1788201706.647698:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.647700:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.647705:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.647713:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.647714:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.647715:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.647721:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.647723:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.647725:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.647727:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647732:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.647738:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.647739:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.647741:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.647743:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116760, 116760] 00000080:00200000:0.0:1788201706.647750:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.647752:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.647769:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.647771:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.647781:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478253056, count: 4096 00000080:00200000:0.0:1788201706.647787:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478253056, bytes: 4096 00000080:00200000:0.0:1788201706.647789:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.647791:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.647792:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478253056, 478257152) 0 0 00000080:00200000:0.0:1788201706.647793:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.647796:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116761, 116761] 00020000:00200000:0.0:1788201706.647798:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478253056 chunk: [478253056, 478257152) 478257152, 4096 00020000:00200000:0.0:1788201706.647800:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.647806:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116761, 116761] 00000008:00010000:0.0:1788201706.647809:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.647810:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.647815:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647824:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478253056 478257151) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647844:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647849:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647853:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.647857:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478253056, 478257152) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.647860:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116761 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.647864:0:17463:0:(rw26.c:898:ll_write_end()) pos 478253056, len 4096, copied 4096 00000080:00200000:0.0:1788201706.647866:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b1e2722e[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116761] 00000080:00200000:0.0:1788201706.647868:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000011e4580c fffffc0001029 2:0 ffff93c0c1331e00 116761 lru 00000080:00200000:0.0:1788201706.647871:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000050994073 116761: 1< 2 - - > 2< 478253056 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68346 - -> 00000080:00200000:0.0:1788201706.647872:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b1e2722e 00000080:00200000:0.0:1788201706.647873:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.647874:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.647880:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.647887:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.647889:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.647891:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.647896:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.647898:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.647899:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.647902:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647906:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.647911:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.647912:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.647915:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.647917:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116761, 116761] 00000080:00200000:0.0:1788201706.647923:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.647925:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.647943:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.647945:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.647955:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478257152, count: 4096 00000080:00200000:0.0:1788201706.647961:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478257152, bytes: 4096 00000080:00200000:0.0:1788201706.647963:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.647965:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.647966:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478257152, 478261248) 0 0 00000080:00200000:0.0:1788201706.647967:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.647970:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116762, 116762] 00020000:00200000:0.0:1788201706.647972:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478257152 chunk: [478257152, 478261248) 478261248, 4096 00020000:00200000:0.0:1788201706.647974:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.647980:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116762, 116762] 00000008:00010000:0.0:1788201706.647983:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.647984:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.647990:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.647995:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478257152 478261247) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648001:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648005:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648009:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.648013:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478257152, 478261248) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.648029:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116762 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.648033:0:17463:0:(rw26.c:898:ll_write_end()) pos 478257152, len 4096, copied 4096 00000080:00200000:0.0:1788201706.648035:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c771a432[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116762] 00000080:00200000:0.0:1788201706.648037:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000489c03cf fffffc0001029 2:0 ffff93c0c13311c0 116762 lru 00000080:00200000:0.0:1788201706.648040:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003922f4da 116762: 1< 2 - - > 2< 478257152 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68347 - -> 00000080:00200000:0.0:1788201706.648042:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c771a432 00000080:00200000:0.0:1788201706.648042:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.648043:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.648049:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.648057:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.648059:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.648060:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.648067:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.648068:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.648070:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.648072:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648077:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.648082:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.648084:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.648086:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.648088:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116762, 116762] 00000080:00200000:0.0:1788201706.648095:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.648097:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.648115:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.648118:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.648128:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478261248, count: 4096 00000080:00200000:0.0:1788201706.648134:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478261248, bytes: 4096 00000080:00200000:0.0:1788201706.648135:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.648138:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.648139:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478261248, 478265344) 0 0 00000080:00200000:0.0:1788201706.648140:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.648144:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116763, 116763] 00020000:00200000:0.0:1788201706.648146:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478261248 chunk: [478261248, 478265344) 478265344, 4096 00020000:00200000:0.0:1788201706.648147:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.648153:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116763, 116763] 00000008:00010000:0.0:1788201706.648157:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.648158:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.648163:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648169:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478261248 478265343) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648174:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648178:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648182:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.648186:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478261248, 478265344) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.648189:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116763 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.648192:0:17463:0:(rw26.c:898:ll_write_end()) pos 478261248, len 4096, copied 4096 00000080:00200000:0.0:1788201706.648194:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000eb828e7b[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116763] 00000080:00200000:0.0:1788201706.648196:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cfd386d3 fffffc0001029 2:0 ffff93c0c1331620 116763 lru 00000080:00200000:0.0:1788201706.648199:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007f13962d 116763: 1< 2 - - > 2< 478261248 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68348 - -> 00000080:00200000:0.0:1788201706.648200:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000eb828e7b 00000080:00200000:0.0:1788201706.648201:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.648202:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.648208:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.648219:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.648220:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.648222:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.648227:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.648229:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.648231:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.648233:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648237:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.648242:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.648244:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.648247:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.648250:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116763, 116763] 00000080:00200000:0.0:1788201706.648260:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.648263:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.648293:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.648297:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.648313:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478265344, count: 4096 00000080:00200000:0.0:1788201706.648324:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478265344, bytes: 4096 00000080:00200000:0.0:1788201706.648326:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.648328:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.648329:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478265344, 478269440) 0 0 00000080:00200000:0.0:1788201706.648331:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.648334:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116764, 116764] 00020000:00200000:0.0:1788201706.648337:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478265344 chunk: [478265344, 478269440) 478269440, 4096 00020000:00200000:0.0:1788201706.648338:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.648345:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116764, 116764] 00000008:00010000:0.0:1788201706.648349:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.648350:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.648356:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648361:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478265344 478269439) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648367:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648372:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648375:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.648380:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478265344, 478269440) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.648383:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116764 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.648387:0:17463:0:(rw26.c:898:ll_write_end()) pos 478265344, len 4096, copied 4096 00000080:00200000:0.0:1788201706.648390:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007740c35f[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116764] 00000080:00200000:0.0:1788201706.648392:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f0a11cff fffffc0001029 2:0 ffff93c0c1331d20 116764 lru 00000080:00200000:0.0:1788201706.648395:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000efcb17c 116764: 1< 2 - - > 2< 478265344 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68349 - -> 00000080:00200000:0.0:1788201706.648397:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007740c35f 00000080:00200000:0.0:1788201706.648397:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.648399:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.648405:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.648413:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.648414:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.648416:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.648422:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.648424:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.648425:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.648428:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648433:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.648438:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.648439:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.648442:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.648443:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116764, 116764] 00000080:00200000:0.0:1788201706.648450:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.648452:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.648470:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.648472:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.648482:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478269440, count: 4096 00000080:00200000:0.0:1788201706.648489:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478269440, bytes: 4096 00000080:00200000:0.0:1788201706.648492:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.648495:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.648497:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478269440, 478273536) 0 0 00000080:00200000:0.0:1788201706.648499:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.648504:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116765, 116765] 00020000:00200000:0.0:1788201706.648507:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478269440 chunk: [478269440, 478273536) 478273536, 4096 00020000:00200000:0.0:1788201706.648509:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.648521:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116765, 116765] 00000008:00010000:0.0:1788201706.648526:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.648528:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.648535:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648543:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478269440 478273535) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648552:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648559:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648565:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.648572:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478269440, 478273536) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.648576:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116765 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.648581:0:17463:0:(rw26.c:898:ll_write_end()) pos 478269440, len 4096, copied 4096 00000080:00200000:0.0:1788201706.648585:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ceaf32a5[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116765] 00000080:00200000:0.0:1788201706.648587:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f7a84807 fffffc0001029 2:0 ffff93c0c13319a0 116765 lru 00000080:00200000:0.0:1788201706.648592:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000463caac 116765: 1< 2 - - > 2< 478269440 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68350 - -> 00000080:00200000:0.0:1788201706.648596:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ceaf32a5 00000080:00200000:0.0:1788201706.648597:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.648599:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.648606:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.648617:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.648620:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.648622:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.648631:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.648633:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.648636:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.648640:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648647:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.648654:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.648656:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.648660:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.648663:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116765, 116765] 00000080:00200000:0.0:1788201706.648673:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.648676:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.648701:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.648704:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.648719:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478273536, count: 4096 00000080:00200000:0.0:1788201706.648728:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478273536, bytes: 4096 00000080:00200000:0.0:1788201706.648731:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.648734:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.648735:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478273536, 478277632) 0 0 00000080:00200000:0.0:1788201706.648738:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.648742:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116766, 116766] 00020000:00200000:0.0:1788201706.648744:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478273536 chunk: [478273536, 478277632) 478277632, 4096 00020000:00200000:0.0:1788201706.648746:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.648756:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116766, 116766] 00000008:00010000:0.0:1788201706.648761:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.648763:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.648770:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648778:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478273536 478277631) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648786:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648792:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648799:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.648805:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478273536, 478277632) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.648809:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116766 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.648814:0:17463:0:(rw26.c:898:ll_write_end()) pos 478273536, len 4096, copied 4096 00000080:00200000:0.0:1788201706.648818:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000034b1076b[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116766] 00000080:00200000:0.0:1788201706.648821:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ea217e3a fffffc0001029 2:0 ffff93c0c13322a0 116766 lru 00000080:00200000:0.0:1788201706.648826:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e3c77130 116766: 1< 2 - - > 2< 478273536 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68351 - -> 00000080:00200000:0.0:1788201706.648828:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000034b1076b 00000080:00200000:0.0:1788201706.648844:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.648847:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.648855:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.648867:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.648870:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.648872:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.648880:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.648883:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.648885:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.648889:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648896:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.648901:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.648902:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.648905:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.648907:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116766, 116766] 00000080:00200000:0.0:1788201706.648914:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.648916:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.648934:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.648937:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.648947:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478277632, count: 4096 00000080:00200000:0.0:1788201706.648953:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478277632, bytes: 4096 00000080:00200000:0.0:1788201706.648955:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.648957:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.648958:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478277632, 478281728) 0 0 00000080:00200000:0.0:1788201706.648960:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.648963:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116767, 116767] 00020000:00200000:0.0:1788201706.648965:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478277632 chunk: [478277632, 478281728) 478281728, 4096 00020000:00200000:0.0:1788201706.648966:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.648973:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116767, 116767] 00000008:00010000:0.0:1788201706.648976:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.648977:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.648982:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648988:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478277632 478281727) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648994:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.648998:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649003:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.649007:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478277632, 478281728) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.649009:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116767 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.649014:0:17463:0:(rw26.c:898:ll_write_end()) pos 478277632, len 4096, copied 4096 00000080:00200000:0.0:1788201706.649031:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e5f9ba3d[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116767] 00000080:00200000:0.0:1788201706.649033:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c52d86d6 fffffc0001029 2:0 ffff93c0c1332e00 116767 lru 00000080:00200000:0.0:1788201706.649036:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ffb4fe30 116767: 1< 2 - - > 2< 478277632 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68352 - -> 00000080:00200000:0.0:1788201706.649038:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e5f9ba3d 00000080:00200000:0.0:1788201706.649039:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.649040:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.649046:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.649054:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.649056:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.649057:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.649064:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.649065:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.649067:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.649070:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649074:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.649079:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.649080:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.649083:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.649085:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116767, 116767] 00000080:00200000:0.0:1788201706.649092:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.649094:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.649112:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.649114:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.649127:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478281728, count: 4096 00000080:00200000:0.0:1788201706.649137:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478281728, bytes: 4096 00000080:00200000:0.0:1788201706.649140:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.649144:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.649145:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478281728, 478285824) 0 0 00000080:00200000:0.0:1788201706.649148:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.649153:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116768, 116768] 00020000:00200000:0.0:1788201706.649156:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478281728 chunk: [478281728, 478285824) 478285824, 4096 00020000:00200000:0.0:1788201706.649158:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.649169:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116768, 116768] 00000008:00010000:0.0:1788201706.649174:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.649177:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.649184:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649193:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478281728 478285823) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649202:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649210:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649217:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.649224:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478281728, 478285824) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.649231:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116768 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.649236:0:17463:0:(rw26.c:898:ll_write_end()) pos 478281728, len 4096, copied 4096 00000080:00200000:0.0:1788201706.649240:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000da59cf6e[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116768] 00000080:00200000:0.0:1788201706.649243:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006b510fae fffffc0001029 2:0 ffff93c0c13321c0 116768 lru 00000080:00200000:0.0:1788201706.649249:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c922b40b 116768: 1< 2 - - > 2< 478281728 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68353 - -> 00000080:00200000:0.0:1788201706.649251:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000da59cf6e 00000080:00200000:0.0:1788201706.649252:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.649254:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.649262:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.649273:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.649276:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.649278:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.649286:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.649289:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.649291:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.649295:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649303:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.649311:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.649314:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.649318:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.649321:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116768, 116768] 00000080:00200000:0.0:1788201706.649332:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.649335:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.649364:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.649367:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.649381:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478285824, count: 4096 00000080:00200000:0.0:1788201706.649389:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478285824, bytes: 4096 00000080:00200000:0.0:1788201706.649392:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.649395:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.649396:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478285824, 478289920) 0 0 00000080:00200000:0.0:1788201706.649398:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.649403:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116769, 116769] 00020000:00200000:0.0:1788201706.649405:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478285824 chunk: [478285824, 478289920) 478289920, 4096 00020000:00200000:0.0:1788201706.649408:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.649418:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116769, 116769] 00000008:00010000:0.0:1788201706.649424:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.649426:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.649433:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649443:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478285824 478289919) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649451:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649459:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649466:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.649477:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478285824, 478289920) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.649481:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116769 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.649485:0:17463:0:(rw26.c:898:ll_write_end()) pos 478285824, len 4096, copied 4096 00000080:00200000:0.0:1788201706.649489:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004ed94fb3[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116769] 00000080:00200000:0.0:1788201706.649492:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000521ebba1 fffffc0001029 2:0 ffff93c0c1332620 116769 lru 00000080:00200000:0.0:1788201706.649497:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006f7ce395 116769: 1< 2 - - > 2< 478285824 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68354 - -> 00000080:00200000:0.0:1788201706.649500:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004ed94fb3 00000080:00200000:0.0:1788201706.649501:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.649516:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.649530:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.649542:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.649545:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.649547:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.649555:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.649556:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.649558:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.649561:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649566:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.649571:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.649572:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.649576:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.649578:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116769, 116769] 00000080:00200000:0.0:1788201706.649585:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.649588:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.649606:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.649609:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.649620:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478289920, count: 4096 00000080:00200000:0.0:1788201706.649626:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478289920, bytes: 4096 00000080:00200000:0.0:1788201706.649627:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.649630:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.649631:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478289920, 478294016) 0 0 00000080:00200000:0.0:1788201706.649632:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.649635:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116770, 116770] 00020000:00200000:0.0:1788201706.649637:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478289920 chunk: [478289920, 478294016) 478294016, 4096 00020000:00200000:0.0:1788201706.649639:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.649646:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116770, 116770] 00000008:00010000:0.0:1788201706.649649:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.649650:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.649656:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649661:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478289920 478294015) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649667:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649671:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649679:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.649683:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478289920, 478294016) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.649686:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116770 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.649690:0:17463:0:(rw26.c:898:ll_write_end()) pos 478289920, len 4096, copied 4096 00000080:00200000:0.0:1788201706.649692:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007522716c[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116770] 00000080:00200000:0.0:1788201706.649694:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007c3b6fd7 fffffc0001029 2:0 ffff93c0c1332d20 116770 lru 00000080:00200000:0.0:1788201706.649697:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007c869f5e 116770: 1< 2 - - > 2< 478289920 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68355 - -> 00000080:00200000:0.0:1788201706.649699:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007522716c 00000080:00200000:0.0:1788201706.649699:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.649701:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.649706:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.649714:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.649715:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.649717:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.649723:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.649724:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.649726:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.649728:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649733:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.649741:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.649743:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.649745:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.649747:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116770, 116770] 00000080:00200000:0.0:1788201706.649754:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.649755:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.649773:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.649775:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.649785:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478294016, count: 4096 00000080:00200000:0.0:1788201706.649791:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478294016, bytes: 4096 00000080:00200000:0.0:1788201706.649793:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.649795:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.649796:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478294016, 478298112) 0 0 00000080:00200000:0.0:1788201706.649798:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.649801:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116771, 116771] 00020000:00200000:0.0:1788201706.649803:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478294016 chunk: [478294016, 478298112) 478298112, 4096 00020000:00200000:0.0:1788201706.649804:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.649810:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116771, 116771] 00000008:00010000:0.0:1788201706.649813:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.649815:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.649820:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649825:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478294016 478298111) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649844:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649853:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649857:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.649862:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478294016, 478298112) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.649865:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116771 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.649869:0:17463:0:(rw26.c:898:ll_write_end()) pos 478294016, len 4096, copied 4096 00000080:00200000:0.0:1788201706.649871:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bc8178bd[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116771] 00000080:00200000:0.0:1788201706.649873:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f2e8de3e fffffc0001029 2:0 ffff93c0c13329a0 116771 lru 00000080:00200000:0.0:1788201706.649876:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000042de6ff6 116771: 1< 2 - - > 2< 478294016 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68356 - -> 00000080:00200000:0.0:1788201706.649877:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bc8178bd 00000080:00200000:0.0:1788201706.649878:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.649879:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.649885:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.649892:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.649894:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.649895:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.649901:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.649903:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.649904:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.649907:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.649915:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.649920:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.649921:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.649923:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.649925:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116771, 116771] 00000080:00200000:0.0:1788201706.649932:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.649934:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.649951:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.649953:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.649963:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478298112, count: 4096 00000080:00200000:0.0:1788201706.649969:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478298112, bytes: 4096 00000080:00200000:0.0:1788201706.649971:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.649973:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.649974:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478298112, 478302208) 0 0 00000080:00200000:0.0:1788201706.649976:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.649979:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116772, 116772] 00020000:00200000:0.0:1788201706.649980:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478298112 chunk: [478298112, 478302208) 478302208, 4096 00020000:00200000:0.0:1788201706.649982:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.649988:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116772, 116772] 00000008:00010000:0.0:1788201706.649991:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.649992:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.649997:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650003:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478298112 478302207) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650012:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650030:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650034:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.650039:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478298112, 478302208) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.650043:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116772 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.650046:0:17463:0:(rw26.c:898:ll_write_end()) pos 478298112, len 4096, copied 4096 00000080:00200000:0.0:1788201706.650049:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000111ae9f8[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116772] 00000080:00200000:0.0:1788201706.650051:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006cf316f7 fffffc0001029 2:0 ffff93c0c1332b60 116772 lru 00000080:00200000:0.0:1788201706.650054:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000021572d47 116772: 1< 2 - - > 2< 478298112 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68357 - -> 00000080:00200000:0.0:1788201706.650056:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000111ae9f8 00000080:00200000:0.0:1788201706.650057:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.650058:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.650064:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.650071:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.650073:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.650074:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.650080:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.650082:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.650084:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.650086:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650091:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.650096:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.650098:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.650100:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.650102:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116772, 116772] 00000080:00200000:0.0:1788201706.650109:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.650111:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.650129:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.650131:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.650141:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478302208, count: 4096 00000080:00200000:0.0:1788201706.650147:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478302208, bytes: 4096 00000080:00200000:0.0:1788201706.650149:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.650151:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.650152:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478302208, 478306304) 0 0 00000080:00200000:0.0:1788201706.650154:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.650157:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116773, 116773] 00020000:00200000:0.0:1788201706.650159:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478302208 chunk: [478302208, 478306304) 478306304, 4096 00020000:00200000:0.0:1788201706.650160:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.650167:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116773, 116773] 00000008:00010000:0.0:1788201706.650170:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.650171:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.650176:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650185:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478302208 478306303) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650191:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650195:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650199:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.650203:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478302208, 478306304) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.650207:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116773 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.650211:0:17463:0:(rw26.c:898:ll_write_end()) pos 478302208, len 4096, copied 4096 00000080:00200000:0.0:1788201706.650213:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009d569b87[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116773] 00000080:00200000:0.0:1788201706.650215:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007fa2e817 fffffc0001029 2:0 ffff93c0c1332c40 116773 lru 00000080:00200000:0.0:1788201706.650218:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b0966b43 116773: 1< 2 - - > 2< 478302208 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68358 - -> 00000080:00200000:0.0:1788201706.650220:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009d569b87 00000080:00200000:0.0:1788201706.650221:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.650222:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.650227:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.650235:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.650236:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.650238:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.650244:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.650246:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.650247:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.650249:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650254:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.650259:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.650260:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.650263:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.650265:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116773, 116773] 00000080:00200000:0.0:1788201706.650271:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.650273:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.650291:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.650293:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.650303:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478306304, count: 4096 00000080:00200000:0.0:1788201706.650309:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478306304, bytes: 4096 00000080:00200000:0.0:1788201706.650311:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.650313:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.650314:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478306304, 478310400) 0 0 00000080:00200000:0.0:1788201706.650315:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.650319:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116774, 116774] 00020000:00200000:0.0:1788201706.650320:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478306304 chunk: [478306304, 478310400) 478310400, 4096 00020000:00200000:0.0:1788201706.650322:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.650328:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116774, 116774] 00000008:00010000:0.0:1788201706.650331:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.650332:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.650337:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650343:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478306304 478310399) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650349:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650353:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650357:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.650361:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478306304, 478310400) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.650364:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116774 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.650368:0:17463:0:(rw26.c:898:ll_write_end()) pos 478306304, len 4096, copied 4096 00000080:00200000:0.0:1788201706.650370:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002203d408[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116774] 00000080:00200000:0.0:1788201706.650372:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b7c6068c fffffc0001029 2:0 ffff93c0c1332540 116774 lru 00000080:00200000:0.0:1788201706.650376:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000260f70aa 116774: 1< 2 - - > 2< 478306304 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68359 - -> 00000080:00200000:0.0:1788201706.650377:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002203d408 00000080:00200000:0.0:1788201706.650378:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.650379:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.650384:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.650392:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.650394:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.650395:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.650401:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.650402:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.650404:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.650406:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650412:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.650417:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.650418:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.650420:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.650422:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116774, 116774] 00000080:00200000:0.0:1788201706.650429:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.650430:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.650448:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.650450:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.650460:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478310400, count: 4096 00000080:00200000:0.0:1788201706.650466:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478310400, bytes: 4096 00000080:00200000:0.0:1788201706.650467:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.650469:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.650470:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478310400, 478314496) 0 0 00000080:00200000:0.0:1788201706.650472:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.650475:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116775, 116775] 00020000:00200000:0.0:1788201706.650477:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478310400 chunk: [478310400, 478314496) 478314496, 4096 00020000:00200000:0.0:1788201706.650478:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.650485:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116775, 116775] 00000008:00010000:0.0:1788201706.650489:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.650491:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.650498:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650507:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478310400 478314495) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650516:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650524:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650531:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.650539:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478310400, 478314496) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.650544:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116775 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.650549:0:17463:0:(rw26.c:898:ll_write_end()) pos 478310400, len 4096, copied 4096 00000080:00200000:0.0:1788201706.650553:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000411c80aa[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116775] 00000080:00200000:0.0:1788201706.650556:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000890fafa5 fffffc0001029 2:0 ffff93c0c1332460 116775 lru 00000080:00200000:0.0:1788201706.650561:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004620cd40 116775: 1< 2 - - > 2< 478310400 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68360 - -> 00000080:00200000:0.0:1788201706.650563:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000411c80aa 00000080:00200000:0.0:1788201706.650563:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.650565:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.650571:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.650582:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.650584:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.650586:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.650592:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.650594:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.650596:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.650598:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650603:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.650608:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.650610:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.650612:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.650614:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116775, 116775] 00000080:00200000:0.0:1788201706.650621:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.650623:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.650641:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.650643:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.650654:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478314496, count: 4096 00000080:00200000:0.0:1788201706.650660:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478314496, bytes: 4096 00000080:00200000:0.0:1788201706.650661:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.650664:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.650665:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478314496, 478318592) 0 0 00000080:00200000:0.0:1788201706.650666:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.650669:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116776, 116776] 00020000:00200000:0.0:1788201706.650672:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478314496 chunk: [478314496, 478318592) 478318592, 4096 00020000:00200000:0.0:1788201706.650673:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.650679:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116776, 116776] 00000008:00010000:0.0:1788201706.650682:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.650684:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.650689:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650695:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478314496 478318591) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650700:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650705:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650708:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.650713:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478314496, 478318592) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.650716:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116776 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.650720:0:17463:0:(rw26.c:898:ll_write_end()) pos 478314496, len 4096, copied 4096 00000080:00200000:0.0:1788201706.650722:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008275e465[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116776] 00000080:00200000:0.0:1788201706.650724:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000aec3acdd fffffc0001029 2:0 ffff93c0c13320e0 116776 lru 00000080:00200000:0.0:1788201706.650727:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c2c8ffd1 116776: 1< 2 - - > 2< 478314496 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68361 - -> 00000080:00200000:0.0:1788201706.650729:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008275e465 00000080:00200000:0.0:1788201706.650729:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.650731:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.650736:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.650744:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.650746:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.650747:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.650752:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.650754:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.650756:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.650758:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650762:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.650767:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.650768:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.650771:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.650773:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116776, 116776] 00000080:00200000:0.0:1788201706.650779:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.650781:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.650799:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.650801:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.650811:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478318592, count: 4096 00000080:00200000:0.0:1788201706.650816:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478318592, bytes: 4096 00000080:00200000:0.0:1788201706.650818:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.650820:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.650821:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478318592, 478322688) 0 0 00000080:00200000:0.0:1788201706.650823:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.650826:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116777, 116777] 00020000:00200000:0.0:1788201706.650840:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478318592 chunk: [478318592, 478322688) 478322688, 4096 00020000:00200000:0.0:1788201706.650842:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.650850:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116777, 116777] 00000008:00010000:0.0:1788201706.650853:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.650855:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.650860:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650866:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478318592 478322687) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650871:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650876:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650880:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.650884:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478318592, 478322688) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.650887:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116777 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.650890:0:17463:0:(rw26.c:898:ll_write_end()) pos 478318592, len 4096, copied 4096 00000080:00200000:0.0:1788201706.650893:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000df357d03[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116777] 00000080:00200000:0.0:1788201706.650894:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006d71433d fffffc0001029 2:0 ffff93c0c1332700 116777 lru 00000080:00200000:0.0:1788201706.650898:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d1be3491 116777: 1< 2 - - > 2< 478318592 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68362 - -> 00000080:00200000:0.0:1788201706.650900:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000df357d03 00000080:00200000:0.0:1788201706.650901:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.650902:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.650908:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.650915:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.650916:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.650918:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.650924:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.650925:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.650927:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.650929:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.650934:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.650939:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.650940:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.650943:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.650945:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116777, 116777] 00000080:00200000:0.0:1788201706.650951:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.650953:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.650971:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.650973:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.650983:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478322688, count: 4096 00000080:00200000:0.0:1788201706.650989:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478322688, bytes: 4096 00000080:00200000:0.0:1788201706.650991:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.650993:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.650994:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478322688, 478326784) 0 0 00000080:00200000:0.0:1788201706.650995:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.650998:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116778, 116778] 00020000:00200000:0.0:1788201706.651000:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478322688 chunk: [478322688, 478326784) 478326784, 4096 00020000:00200000:0.0:1788201706.651002:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.651008:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116778, 116778] 00000008:00010000:0.0:1788201706.651011:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.651012:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.651030:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651036:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478322688 478326783) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651042:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651046:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651050:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.651055:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478322688, 478326784) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.651059:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116778 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.651062:0:17463:0:(rw26.c:898:ll_write_end()) pos 478322688, len 4096, copied 4096 00000080:00200000:0.0:1788201706.651065:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005cc758eb[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116778] 00000080:00200000:0.0:1788201706.651067:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000267842b5 fffffc0001029 2:0 ffff93c0c1332000 116778 lru 00000080:00200000:0.0:1788201706.651070:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ccfc0948 116778: 1< 2 - - > 2< 478322688 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68363 - -> 00000080:00200000:0.0:1788201706.651072:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005cc758eb 00000080:00200000:0.0:1788201706.651073:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.651074:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.651080:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.651087:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.651088:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.651090:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.651096:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.651097:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.651099:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.651101:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651106:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.651111:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.651113:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.651115:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.651117:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116778, 116778] 00000080:00200000:0.0:1788201706.651124:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.651126:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.651144:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.651146:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.651157:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478326784, count: 4096 00000080:00200000:0.0:1788201706.651162:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478326784, bytes: 4096 00000080:00200000:0.0:1788201706.651164:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.651166:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.651167:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478326784, 478330880) 0 0 00000080:00200000:0.0:1788201706.651169:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.651172:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116779, 116779] 00020000:00200000:0.0:1788201706.651174:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478326784 chunk: [478326784, 478330880) 478330880, 4096 00020000:00200000:0.0:1788201706.651175:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.651181:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116779, 116779] 00000008:00010000:0.0:1788201706.651184:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.651186:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.651191:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651197:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478326784 478330879) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651202:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651206:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651210:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.651215:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478326784, 478330880) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.651218:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116779 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.651222:0:17463:0:(rw26.c:898:ll_write_end()) pos 478326784, len 4096, copied 4096 00000080:00200000:0.0:1788201706.651224:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f1074c22[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116779] 00000080:00200000:0.0:1788201706.651226:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000dcfabfef fffffc0001029 2:0 ffff93c0c1332a80 116779 lru 00000080:00200000:0.0:1788201706.651230:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d2d3f209 116779: 1< 2 - - > 2< 478326784 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68364 - -> 00000080:00200000:0.0:1788201706.651231:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f1074c22 00000080:00200000:0.0:1788201706.651232:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.651233:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.651238:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.651246:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.651248:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.651249:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.651255:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.651256:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.651258:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.651260:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651265:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.651269:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.651271:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.651273:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.651276:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116779, 116779] 00000080:00200000:0.0:1788201706.651282:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.651284:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.651302:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.651304:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.651314:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478330880, count: 4096 00000080:00200000:0.0:1788201706.651320:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478330880, bytes: 4096 00000080:00200000:0.0:1788201706.651321:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.651323:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.651324:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478330880, 478334976) 0 0 00000080:00200000:0.0:1788201706.651326:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.651329:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116780, 116780] 00020000:00200000:0.0:1788201706.651331:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478330880 chunk: [478330880, 478334976) 478334976, 4096 00020000:00200000:0.0:1788201706.651332:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.651339:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116780, 116780] 00000008:00010000:0.0:1788201706.651341:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.651343:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.651348:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651353:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478330880 478334975) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651359:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651363:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651367:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.651372:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478330880, 478334976) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.651375:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116780 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.651379:0:17463:0:(rw26.c:898:ll_write_end()) pos 478330880, len 4096, copied 4096 00000080:00200000:0.0:1788201706.651381:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000163da6b[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116780] 00000080:00200000:0.0:1788201706.651383:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a4e85b47 fffffc0001029 2:0 ffff93c0c1332380 116780 lru 00000080:00200000:0.0:1788201706.651386:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000db9a6b45 116780: 1< 2 - - > 2< 478330880 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68365 - -> 00000080:00200000:0.0:1788201706.651388:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000163da6b 00000080:00200000:0.0:1788201706.651388:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.651390:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.651395:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.651402:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.651404:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.651405:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.651411:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.651413:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.651414:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.651417:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651421:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.651425:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.651427:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.651431:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.651433:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116780, 116780] 00000080:00200000:0.0:1788201706.651443:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.651446:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.651474:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.651477:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.651492:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478334976, count: 4096 00000080:00200000:0.0:1788201706.651503:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478334976, bytes: 4096 00000080:00200000:0.0:1788201706.651506:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.651510:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.651512:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478334976, 478339072) 0 0 00000080:00200000:0.0:1788201706.651515:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.651520:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116781, 116781] 00020000:00200000:0.0:1788201706.651523:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478334976 chunk: [478334976, 478339072) 478339072, 4096 00020000:00200000:0.0:1788201706.651526:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.651537:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116781, 116781] 00000008:00010000:0.0:1788201706.651544:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.651545:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.651551:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651557:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478334976 478339071) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651562:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651567:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651571:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.651579:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478334976, 478339072) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.651582:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116781 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.651586:0:17463:0:(rw26.c:898:ll_write_end()) pos 478334976, len 4096, copied 4096 00000080:00200000:0.0:1788201706.651589:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000099ccd5e4[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116781] 00000080:00200000:0.0:1788201706.651590:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f625ec52 fffffc0001029 2:0 ffff93c0c13328c0 116781 lru 00000080:00200000:0.0:1788201706.651594:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d618a574 116781: 1< 2 - - > 2< 478334976 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68366 - -> 00000080:00200000:0.0:1788201706.651596:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000099ccd5e4 00000080:00200000:0.0:1788201706.651596:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.651597:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.651603:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.651611:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.651612:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.651614:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.651620:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.651622:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.651624:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.651626:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651631:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.651636:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.651637:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.651640:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.651642:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116781, 116781] 00000080:00200000:0.0:1788201706.651649:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.651651:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.651670:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.651672:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.651682:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478339072, count: 4096 00000080:00200000:0.0:1788201706.651688:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478339072, bytes: 4096 00000080:00200000:0.0:1788201706.651690:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.651692:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.651693:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478339072, 478343168) 0 0 00000080:00200000:0.0:1788201706.651695:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.651698:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116782, 116782] 00020000:00200000:0.0:1788201706.651700:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478339072 chunk: [478339072, 478343168) 478343168, 4096 00020000:00200000:0.0:1788201706.651701:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.651708:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116782, 116782] 00000008:00010000:0.0:1788201706.651711:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.651712:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.651717:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651722:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478339072 478343167) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651728:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651732:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651740:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.651744:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478339072, 478343168) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.651747:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116782 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.651750:0:17463:0:(rw26.c:898:ll_write_end()) pos 478339072, len 4096, copied 4096 00000080:00200000:0.0:1788201706.651753:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000686a0c0f[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116782] 00000080:00200000:0.0:1788201706.651754:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009104fd9c fffffc0001029 2:0 ffff93c0c1332ee0 116782 lru 00000080:00200000:0.0:1788201706.651758:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000035ceb696 116782: 1< 2 - - > 2< 478339072 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68367 - -> 00000080:00200000:0.0:1788201706.651759:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000686a0c0f 00000080:00200000:0.0:1788201706.651760:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.651761:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.651767:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.651774:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.651776:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.651777:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.651783:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.651784:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.651786:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.651788:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651793:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.651801:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.651803:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.651805:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.651807:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116782, 116782] 00000080:00200000:0.0:1788201706.651814:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.651815:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.651853:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.651855:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.651866:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478343168, count: 4096 00000080:00200000:0.0:1788201706.651873:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478343168, bytes: 4096 00000080:00200000:0.0:1788201706.651875:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.651878:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.651879:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478343168, 478347264) 0 0 00000080:00200000:0.0:1788201706.651880:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.651883:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116783, 116783] 00020000:00200000:0.0:1788201706.651885:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478343168 chunk: [478343168, 478347264) 478347264, 4096 00020000:00200000:0.0:1788201706.651887:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.651893:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116783, 116783] 00000008:00010000:0.0:1788201706.651896:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.651898:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.651903:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651908:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478343168 478347263) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651914:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651922:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651926:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.651930:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478343168, 478347264) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.651933:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116783 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.651937:0:17463:0:(rw26.c:898:ll_write_end()) pos 478343168, len 4096, copied 4096 00000080:00200000:0.0:1788201706.651939:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006c943633[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116783] 00000080:00200000:0.0:1788201706.651941:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004cd9275c fffffc0001029 2:0 ffff93c0c13327e0 116783 lru 00000080:00200000:0.0:1788201706.651944:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006c97f297 116783: 1< 2 - - > 2< 478343168 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68368 - -> 00000080:00200000:0.0:1788201706.651945:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006c943633 00000080:00200000:0.0:1788201706.651946:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.651947:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.651953:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.651960:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.651962:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.651963:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.651969:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.651971:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.651972:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.651975:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.651983:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.651988:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.651989:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.651991:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.651993:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116783, 116783] 00000080:00200000:0.0:1788201706.652000:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.652001:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.652033:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.652036:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.652047:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478347264, count: 4096 00000080:00200000:0.0:1788201706.652053:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478347264, bytes: 4096 00000080:00200000:0.0:1788201706.652055:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.652058:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.652059:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478347264, 478351360) 0 0 00000080:00200000:0.0:1788201706.652060:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.652063:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116784, 116784] 00020000:00200000:0.0:1788201706.652066:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478347264 chunk: [478347264, 478351360) 478351360, 4096 00020000:00200000:0.0:1788201706.652067:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.652073:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116784, 116784] 00000008:00010000:0.0:1788201706.652077:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.652078:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.652083:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652089:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478347264 478351359) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652098:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652102:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652106:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.652111:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478347264, 478351360) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.652114:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116784 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.652118:0:17463:0:(rw26.c:898:ll_write_end()) pos 478347264, len 4096, copied 4096 00000080:00200000:0.0:1788201706.652120:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000139a7123[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116784] 00000080:00200000:0.0:1788201706.652122:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008bbc1573 fffffc0001029 2:0 ffff93c0c1333620 116784 lru 00000080:00200000:0.0:1788201706.652126:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000ee01f61 116784: 1< 2 - - > 2< 478347264 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68369 - -> 00000080:00200000:0.0:1788201706.652127:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000139a7123 00000080:00200000:0.0:1788201706.652128:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.652129:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.652135:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.652142:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.652144:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.652145:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.652151:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.652153:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.652155:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.652157:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652162:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.652167:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.652168:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.652171:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.652173:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116784, 116784] 00000080:00200000:0.0:1788201706.652180:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.652181:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.652199:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.652201:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.652211:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478351360, count: 4096 00000080:00200000:0.0:1788201706.652217:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478351360, bytes: 4096 00000080:00200000:0.0:1788201706.652219:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.652221:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.652222:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478351360, 478355456) 0 0 00000080:00200000:0.0:1788201706.652224:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.652227:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116785, 116785] 00020000:00200000:0.0:1788201706.652229:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478351360 chunk: [478351360, 478355456) 478355456, 4096 00020000:00200000:0.0:1788201706.652230:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.652236:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116785, 116785] 00000008:00010000:0.0:1788201706.652239:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.652241:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.652246:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652254:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478351360 478355455) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652260:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652264:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652268:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.652273:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478351360, 478355456) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.652276:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116785 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.652279:0:17463:0:(rw26.c:898:ll_write_end()) pos 478351360, len 4096, copied 4096 00000080:00200000:0.0:1788201706.652282:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000011752561[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116785] 00000080:00200000:0.0:1788201706.652284:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b7efc94c fffffc0001029 2:0 ffff93c0c1333d20 116785 lru 00000080:00200000:0.0:1788201706.652287:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a8b8919b 116785: 1< 2 - - > 2< 478351360 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68370 - -> 00000080:00200000:0.0:1788201706.652289:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000011752561 00000080:00200000:0.0:1788201706.652290:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.652291:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.652296:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.652303:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.652305:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.652307:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.652312:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.652314:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.652316:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.652318:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652323:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.652327:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.652329:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.652331:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.652333:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116785, 116785] 00000080:00200000:0.0:1788201706.652339:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.652341:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.652359:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.652361:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.652371:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478355456, count: 4096 00000080:00200000:0.0:1788201706.652377:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478355456, bytes: 4096 00000080:00200000:0.0:1788201706.652378:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.652380:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.652381:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478355456, 478359552) 0 0 00000080:00200000:0.0:1788201706.652383:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.652386:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116786, 116786] 00020000:00200000:0.0:1788201706.652388:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478355456 chunk: [478355456, 478359552) 478359552, 4096 00020000:00200000:0.0:1788201706.652389:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.652395:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116786, 116786] 00000008:00010000:0.0:1788201706.652398:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.652400:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.652405:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652410:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478355456 478359551) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652416:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652420:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652424:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.652430:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478355456, 478359552) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.652435:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116786 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.652440:0:17463:0:(rw26.c:898:ll_write_end()) pos 478355456, len 4096, copied 4096 00000080:00200000:0.0:1788201706.652444:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008dec02ca[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116786] 00000080:00200000:0.0:1788201706.652447:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000071912d16 fffffc0001029 2:0 ffff93c0c13339a0 116786 lru 00000080:00200000:0.0:1788201706.652452:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000068c2b9f9 116786: 1< 2 - - > 2< 478355456 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68371 - -> 00000080:00200000:0.0:1788201706.652455:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008dec02ca 00000080:00200000:0.0:1788201706.652456:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.652458:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.652466:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.652478:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.652482:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.652483:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.652492:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.652495:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.652497:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.652502:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652510:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.652518:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.652520:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.652525:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.652528:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116786, 116786] 00000080:00200000:0.0:1788201706.652539:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.652542:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.652571:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.652575:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.652589:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478359552, count: 4096 00000080:00200000:0.0:1788201706.652596:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478359552, bytes: 4096 00000080:00200000:0.0:1788201706.652598:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.652600:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.652601:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478359552, 478363648) 0 0 00000080:00200000:0.0:1788201706.652603:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.652606:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116787, 116787] 00020000:00200000:0.0:1788201706.652608:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478359552 chunk: [478359552, 478363648) 478363648, 4096 00020000:00200000:0.0:1788201706.652610:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.652617:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116787, 116787] 00000008:00010000:0.0:1788201706.652620:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.652622:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.652627:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652633:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478359552 478363647) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652639:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652643:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652647:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.652652:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478359552, 478363648) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.652655:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116787 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.652659:0:17463:0:(rw26.c:898:ll_write_end()) pos 478359552, len 4096, copied 4096 00000080:00200000:0.0:1788201706.652661:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fb127b9f[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116787] 00000080:00200000:0.0:1788201706.652663:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000965ac511 fffffc0001029 2:0 ffff93c0c1333b60 116787 lru 00000080:00200000:0.0:1788201706.652667:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008ee2971e 116787: 1< 2 - - > 2< 478359552 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68372 - -> 00000080:00200000:0.0:1788201706.652668:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fb127b9f 00000080:00200000:0.0:1788201706.652669:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.652670:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.652676:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.652687:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.652688:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.652690:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.652696:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.652698:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.652699:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.652702:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652707:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.652712:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.652713:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.652715:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.652717:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116787, 116787] 00000080:00200000:0.0:1788201706.652724:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.652726:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.652744:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.652746:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.652756:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478363648, count: 4096 00000080:00200000:0.0:1788201706.652762:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478363648, bytes: 4096 00000080:00200000:0.0:1788201706.652764:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.652766:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.652767:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478363648, 478367744) 0 0 00000080:00200000:0.0:1788201706.652769:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.652772:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116788, 116788] 00020000:00200000:0.0:1788201706.652774:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478363648 chunk: [478363648, 478367744) 478367744, 4096 00020000:00200000:0.0:1788201706.652775:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.652782:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116788, 116788] 00000008:00010000:0.0:1788201706.652785:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.652786:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.652791:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652797:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478363648 478367743) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652802:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652806:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652810:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.652815:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478363648, 478367744) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.652818:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116788 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.652822:0:17463:0:(rw26.c:898:ll_write_end()) pos 478363648, len 4096, copied 4096 00000080:00200000:0.0:1788201706.652824:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004758413c[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116788] 00000080:00200000:0.0:1788201706.652826:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cb0f201a fffffc0001029 2:0 ffff93c0c1333c40 116788 lru 00000080:00200000:0.0:1788201706.652843:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006bd1eaf6 116788: 1< 2 - - > 2< 478363648 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68373 - -> 00000080:00200000:0.0:1788201706.652845:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004758413c 00000080:00200000:0.0:1788201706.652846:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.652847:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.652853:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.652861:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.652863:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.652864:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.652870:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.652872:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.652873:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.652876:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652881:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.652886:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.652887:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.652889:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.652891:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116788, 116788] 00000080:00200000:0.0:1788201706.652898:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.652899:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.652917:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.652919:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.652929:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478367744, count: 4096 00000080:00200000:0.0:1788201706.652935:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478367744, bytes: 4096 00000080:00200000:0.0:1788201706.652936:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.652938:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.652939:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478367744, 478371840) 0 0 00000080:00200000:0.0:1788201706.652941:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.652944:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116789, 116789] 00020000:00200000:0.0:1788201706.652946:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478367744 chunk: [478367744, 478371840) 478371840, 4096 00020000:00200000:0.0:1788201706.652948:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.652954:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116789, 116789] 00000008:00010000:0.0:1788201706.652957:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.652959:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.652964:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652970:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478367744 478371839) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652975:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652980:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.652984:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.652988:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478367744, 478371840) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.652991:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116789 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.652995:0:17463:0:(rw26.c:898:ll_write_end()) pos 478367744, len 4096, copied 4096 00000080:00200000:0.0:1788201706.652997:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b44c31ab[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116789] 00000080:00200000:0.0:1788201706.653000:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d2a009dc fffffc0001029 2:0 ffff93c0c137e1c0 116789 lru 00000080:00200000:0.0:1788201706.653003:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000088adad94 116789: 1< 2 - - > 2< 478367744 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68374 - -> 00000080:00200000:0.0:1788201706.653006:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b44c31ab 00000080:00200000:0.0:1788201706.653006:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.653007:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.653013:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.653035:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.653037:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.653039:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.653045:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.653047:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.653049:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.653051:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653056:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.653061:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.653062:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.653065:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.653066:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116789, 116789] 00000080:00200000:0.0:1788201706.653074:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.653075:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.653093:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.653096:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.653106:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478371840, count: 4096 00000080:00200000:0.0:1788201706.653112:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478371840, bytes: 4096 00000080:00200000:0.0:1788201706.653114:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.653116:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.653117:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478371840, 478375936) 0 0 00000080:00200000:0.0:1788201706.653118:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.653122:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116790, 116790] 00020000:00200000:0.0:1788201706.653124:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478371840 chunk: [478371840, 478375936) 478375936, 4096 00020000:00200000:0.0:1788201706.653125:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.653132:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116790, 116790] 00000008:00010000:0.0:1788201706.653135:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.653136:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.653141:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653147:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478371840 478375935) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653152:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653156:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653160:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.653165:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478371840, 478375936) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.653168:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116790 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.653171:0:17463:0:(rw26.c:898:ll_write_end()) pos 478371840, len 4096, copied 4096 00000080:00200000:0.0:1788201706.653174:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000005b2cf16[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116790] 00000080:00200000:0.0:1788201706.653176:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e28521c1 fffffc0001029 2:0 ffff93c0c137e620 116790 lru 00000080:00200000:0.0:1788201706.653179:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000070e8611f 116790: 1< 2 - - > 2< 478371840 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68375 - -> 00000080:00200000:0.0:1788201706.653180:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000005b2cf16 00000080:00200000:0.0:1788201706.653181:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.653182:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.653187:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.653194:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.653196:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.653197:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.653203:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.653204:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.653206:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.653208:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653213:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.653218:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.653220:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.653222:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.653224:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116790, 116790] 00000080:00200000:0.0:1788201706.653231:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.653233:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.653250:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.653252:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.653262:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478375936, count: 4096 00000080:00200000:0.0:1788201706.653268:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478375936, bytes: 4096 00000080:00200000:0.0:1788201706.653270:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.653272:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.653273:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478375936, 478380032) 0 0 00000080:00200000:0.0:1788201706.653275:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.653278:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116791, 116791] 00020000:00200000:0.0:1788201706.653280:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478375936 chunk: [478375936, 478380032) 478380032, 4096 00020000:00200000:0.0:1788201706.653281:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.653288:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116791, 116791] 00000008:00010000:0.0:1788201706.653291:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.653292:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.653297:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653302:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478375936 478380031) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653308:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653313:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653316:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.653320:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478375936, 478380032) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.653323:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116791 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.653327:0:17463:0:(rw26.c:898:ll_write_end()) pos 478375936, len 4096, copied 4096 00000080:00200000:0.0:1788201706.653329:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006383857f[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116791] 00000080:00200000:0.0:1788201706.653331:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006eaf7ce2 fffffc0001029 2:0 ffff93c0c137ed20 116791 lru 00000080:00200000:0.0:1788201706.653335:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b2007c1c 116791: 1< 2 - - > 2< 478375936 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68376 - -> 00000080:00200000:0.0:1788201706.653336:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006383857f 00000080:00200000:0.0:1788201706.653337:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.653338:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.653344:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.653351:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.653353:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.653354:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.653360:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.653362:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.653363:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.653366:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653370:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.653375:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.653376:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.653379:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.653381:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116791, 116791] 00000080:00200000:0.0:1788201706.653387:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.653389:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.653406:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.653409:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.653419:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478380032, count: 4096 00000080:00200000:0.0:1788201706.653425:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478380032, bytes: 4096 00000080:00200000:0.0:1788201706.653427:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.653430:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.653431:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478380032, 478384128) 0 0 00000080:00200000:0.0:1788201706.653434:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.653438:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116792, 116792] 00020000:00200000:0.0:1788201706.653441:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478380032 chunk: [478380032, 478384128) 478384128, 4096 00020000:00200000:0.0:1788201706.653443:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.653454:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116792, 116792] 00000008:00010000:0.0:1788201706.653459:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.653461:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.653469:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653477:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478380032 478384127) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653485:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653492:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653499:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.653507:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478380032, 478384128) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.653512:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116792 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.653518:0:17463:0:(rw26.c:898:ll_write_end()) pos 478380032, len 4096, copied 4096 00000080:00200000:0.0:1788201706.653522:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000071afac99[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116792] 00000080:00200000:0.0:1788201706.653524:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008b6c02ed fffffc0001029 2:0 ffff93c0c137e9a0 116792 lru 00000080:00200000:0.0:1788201706.653529:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000049a5c42b 116792: 1< 2 - - > 2< 478380032 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68377 - -> 00000080:00200000:0.0:1788201706.653531:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000071afac99 00000080:00200000:0.0:1788201706.653532:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.653534:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.653542:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.653554:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.653556:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.653558:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.653567:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.653570:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.653572:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.653576:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653584:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.653590:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.653593:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.653597:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.653600:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116792, 116792] 00000080:00200000:0.0:1788201706.653610:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.653613:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.653639:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.653642:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.653657:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478384128, count: 4096 00000080:00200000:0.0:1788201706.653667:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478384128, bytes: 4096 00000080:00200000:0.0:1788201706.653670:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.653673:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.653675:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478384128, 478388224) 0 0 00000080:00200000:0.0:1788201706.653677:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.653682:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116793, 116793] 00020000:00200000:0.0:1788201706.653685:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478384128 chunk: [478384128, 478388224) 478388224, 4096 00020000:00200000:0.0:1788201706.653687:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.653699:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116793, 116793] 00000008:00010000:0.0:1788201706.653704:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.653706:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.653713:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653721:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478384128 478388223) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653729:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653736:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653743:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.653756:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478384128, 478388224) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.653760:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116793 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.653764:0:17463:0:(rw26.c:898:ll_write_end()) pos 478384128, len 4096, copied 4096 00000080:00200000:0.0:1788201706.653766:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e0d9ac4f[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116793] 00000080:00200000:0.0:1788201706.653768:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b0b52c38 fffffc0001029 2:0 ffff93c0c137eb60 116793 lru 00000080:00200000:0.0:1788201706.653772:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a9dd795b 116793: 1< 2 - - > 2< 478384128 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68378 - -> 00000080:00200000:0.0:1788201706.653774:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e0d9ac4f 00000080:00200000:0.0:1788201706.653774:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.653776:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.653782:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.653789:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.653791:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.653792:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.653799:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.653801:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.653802:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.653805:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653810:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.653815:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.653817:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.653819:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.653821:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116793, 116793] 00000080:00200000:0.0:1788201706.653845:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.653848:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.653871:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.653873:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.653884:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478388224, count: 4096 00000080:00200000:0.0:1788201706.653890:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478388224, bytes: 4096 00000080:00200000:0.0:1788201706.653892:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.653894:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.653895:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478388224, 478392320) 0 0 00000080:00200000:0.0:1788201706.653897:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.653900:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116794, 116794] 00020000:00200000:0.0:1788201706.653902:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478388224 chunk: [478388224, 478392320) 478392320, 4096 00020000:00200000:0.0:1788201706.653904:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.653910:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116794, 116794] 00000008:00010000:0.0:1788201706.653913:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.653915:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.653920:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653926:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478388224 478392319) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653931:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653935:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653943:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.653947:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478388224, 478392320) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.653950:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116794 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.653954:0:17463:0:(rw26.c:898:ll_write_end()) pos 478388224, len 4096, copied 4096 00000080:00200000:0.0:1788201706.653956:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000053790268[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116794] 00000080:00200000:0.0:1788201706.653957:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006e1bc825 fffffc0001029 2:0 ffff93c0c137ec40 116794 lru 00000080:00200000:0.0:1788201706.653961:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002a058cf8 116794: 1< 2 - - > 2< 478388224 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68379 - -> 00000080:00200000:0.0:1788201706.653962:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000053790268 00000080:00200000:0.0:1788201706.653963:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.653964:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.653970:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.653977:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.653978:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.653980:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.653986:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.653987:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.653989:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.653991:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.653996:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.654004:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.654006:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.654008:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.654010:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116794, 116794] 00000080:00200000:0.0:1788201706.654032:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.654035:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.654054:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.654056:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.654066:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478392320, count: 4096 00000080:00200000:0.0:1788201706.654072:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478392320, bytes: 4096 00000080:00200000:0.0:1788201706.654074:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.654076:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.654077:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478392320, 478396416) 0 0 00000080:00200000:0.0:1788201706.654079:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.654082:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116795, 116795] 00020000:00200000:0.0:1788201706.654084:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478392320 chunk: [478392320, 478396416) 478396416, 4096 00020000:00200000:0.0:1788201706.654086:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.654093:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116795, 116795] 00000008:00010000:0.0:1788201706.654096:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.654097:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.654103:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654109:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478392320 478396415) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654114:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654122:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654126:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.654130:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478392320, 478396416) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.654133:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116795 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.654137:0:17463:0:(rw26.c:898:ll_write_end()) pos 478392320, len 4096, copied 4096 00000080:00200000:0.0:1788201706.654139:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c9e65970[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116795] 00000080:00200000:0.0:1788201706.654141:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cc330a7d fffffc0001029 2:0 ffff93c0c137e540 116795 lru 00000080:00200000:0.0:1788201706.654144:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000fc951220 116795: 1< 2 - - > 2< 478392320 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68380 - -> 00000080:00200000:0.0:1788201706.654145:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c9e65970 00000080:00200000:0.0:1788201706.654146:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.654147:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.654153:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.654160:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.654162:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.654163:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.654169:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.654171:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.654172:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.654175:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654183:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.654187:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.654189:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.654191:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.654193:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116795, 116795] 00000080:00200000:0.0:1788201706.654199:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.654201:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.654219:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.654221:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.654231:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478396416, count: 4096 00000080:00200000:0.0:1788201706.654237:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478396416, bytes: 4096 00000080:00200000:0.0:1788201706.654238:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.654240:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.654241:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478396416, 478400512) 0 0 00000080:00200000:0.0:1788201706.654243:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.654246:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116796, 116796] 00020000:00200000:0.0:1788201706.654248:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478396416 chunk: [478396416, 478400512) 478400512, 4096 00020000:00200000:0.0:1788201706.654249:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.654255:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116796, 116796] 00000008:00010000:0.0:1788201706.654258:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.654260:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.654265:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654270:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478396416 478400511) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654279:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654283:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654287:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.654292:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478396416, 478400512) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.654295:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116796 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.654298:0:17463:0:(rw26.c:898:ll_write_end()) pos 478396416, len 4096, copied 4096 00000080:00200000:0.0:1788201706.654301:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000368e673b[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116796] 00000080:00200000:0.0:1788201706.654303:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000016510998 fffffc0001029 2:0 ffff93c0c137e460 116796 lru 00000080:00200000:0.0:1788201706.654306:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000073794bd5 116796: 1< 2 - - > 2< 478396416 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68381 - -> 00000080:00200000:0.0:1788201706.654308:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000368e673b 00000080:00200000:0.0:1788201706.654309:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.654310:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.654315:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.654323:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.654324:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.654325:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.654331:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.654333:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.654335:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.654337:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654342:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.654347:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.654349:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.654351:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.654353:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116796, 116796] 00000080:00200000:0.0:1788201706.654360:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.654361:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.654379:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.654381:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.654391:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478400512, count: 4096 00000080:00200000:0.0:1788201706.654397:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478400512, bytes: 4096 00000080:00200000:0.0:1788201706.654398:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.654400:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.654402:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478400512, 478404608) 0 0 00000080:00200000:0.0:1788201706.654403:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.654406:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116797, 116797] 00020000:00200000:0.0:1788201706.654408:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478400512 chunk: [478400512, 478404608) 478404608, 4096 00020000:00200000:0.0:1788201706.654409:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.654415:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116797, 116797] 00000008:00010000:0.0:1788201706.654418:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.654420:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.654425:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654437:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478400512 478404607) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654445:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654453:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654460:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.654468:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478400512, 478404608) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.654474:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116797 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.654479:0:17463:0:(rw26.c:898:ll_write_end()) pos 478400512, len 4096, copied 4096 00000080:00200000:0.0:1788201706.654482:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000872c4ed9[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116797] 00000080:00200000:0.0:1788201706.654484:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000de5351dd fffffc0001029 2:0 ffff93c0c137f0e0 116797 lru 00000080:00200000:0.0:1788201706.654488:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006e798595 116797: 1< 2 - - > 2< 478400512 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68382 - -> 00000080:00200000:0.0:1788201706.654490:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000872c4ed9 00000080:00200000:0.0:1788201706.654491:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.654493:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.654502:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.654513:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.654516:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.654519:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.654528:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.654530:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.654533:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.654536:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654542:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.654548:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.654549:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.654552:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.654554:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116797, 116797] 00000080:00200000:0.0:1788201706.654562:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.654564:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.654582:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.654584:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.654595:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478404608, count: 4096 00000080:00200000:0.0:1788201706.654601:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478404608, bytes: 4096 00000080:00200000:0.0:1788201706.654602:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.654605:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.654606:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478404608, 478408704) 0 0 00000080:00200000:0.0:1788201706.654607:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.654611:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116798, 116798] 00020000:00200000:0.0:1788201706.654613:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478404608 chunk: [478404608, 478408704) 478408704, 4096 00020000:00200000:0.0:1788201706.654614:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.654621:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116798, 116798] 00000008:00010000:0.0:1788201706.654624:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.654626:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.654631:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654637:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478404608 478408703) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654642:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654647:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654651:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.654655:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478404608, 478408704) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.654658:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116798 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.654662:0:17463:0:(rw26.c:898:ll_write_end()) pos 478404608, len 4096, copied 4096 00000080:00200000:0.0:1788201706.654664:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000013b62dda[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116798] 00000080:00200000:0.0:1788201706.654666:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007f31e495 fffffc0001029 2:0 ffff93c0c137f700 116798 lru 00000080:00200000:0.0:1788201706.654670:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f821f204 116798: 1< 2 - - > 2< 478404608 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68383 - -> 00000080:00200000:0.0:1788201706.654671:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000013b62dda 00000080:00200000:0.0:1788201706.654672:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.654673:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.654679:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.654686:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.654688:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.654689:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.654695:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.654697:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.654698:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.654701:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654705:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.654710:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.654711:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.654714:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.654716:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116798, 116798] 00000080:00200000:0.0:1788201706.654722:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.654724:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.654742:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.654745:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.654754:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478408704, count: 4096 00000080:00200000:0.0:1788201706.654761:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478408704, bytes: 4096 00000080:00200000:0.0:1788201706.654762:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.654764:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.654766:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478408704, 478412800) 0 0 00000080:00200000:0.0:1788201706.654767:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.654770:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116799, 116799] 00020000:00200000:0.0:1788201706.654772:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478408704 chunk: [478408704, 478412800) 478412800, 4096 00020000:00200000:0.0:1788201706.654773:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.654780:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116799, 116799] 00000008:00010000:0.0:1788201706.654783:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.654785:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.654790:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654795:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478408704 478412799) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654801:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654805:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654809:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.654813:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478408704, 478412800) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.654816:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116799 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.654820:0:17463:0:(rw26.c:898:ll_write_end()) pos 478408704, len 4096, copied 4096 00000080:00200000:0.0:1788201706.654822:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000070279c29[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116799] 00000080:00200000:0.0:1788201706.654823:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e0b4a76f fffffc0001029 2:0 ffff93c0c137f000 116799 lru 00000080:00200000:0.0:1788201706.654827:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005992be19 116799: 1< 2 - - > 2< 478408704 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68384 - -> 00000080:00200000:0.0:1788201706.654841:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000070279c29 00000080:00200000:0.0:1788201706.654842:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.654844:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.654850:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.654861:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.654862:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.654864:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.654870:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.654871:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.654873:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.654876:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654881:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.654886:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.654887:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.654890:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.654891:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116799, 116799] 00000080:00200000:0.0:1788201706.654898:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.654900:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.654917:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.654920:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.654930:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478412800, count: 4096 00000080:00200000:0.0:1788201706.654936:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478412800, bytes: 4096 00000080:00200000:0.0:1788201706.654937:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.654939:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.654941:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478412800, 478416896) 0 0 00000080:00200000:0.0:1788201706.654942:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.654946:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116800, 116800] 00020000:00200000:0.0:1788201706.654948:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478412800 chunk: [478412800, 478416896) 478416896, 4096 00020000:00200000:0.0:1788201706.654950:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.654956:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116800, 116800] 00000008:00010000:0.0:1788201706.654959:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.654960:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.654965:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654971:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478412800 478416895) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654976:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654980:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.654984:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.654989:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478412800, 478416896) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.654992:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116800 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.654995:0:17463:0:(rw26.c:898:ll_write_end()) pos 478412800, len 4096, copied 4096 00000080:00200000:0.0:1788201706.654998:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f21826c9[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116800] 00000080:00200000:0.0:1788201706.655000:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005fb308e9 fffffc0001029 2:0 ffff93c0c137fa80 116800 lru 00000080:00200000:0.0:1788201706.655003:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000fc8a5b21 116800: 1< 2 - - > 2< 478412800 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68385 - -> 00000080:00200000:0.0:1788201706.655005:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f21826c9 00000080:00200000:0.0:1788201706.655005:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.655007:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.655012:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.655036:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.655039:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.655040:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.655049:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.655051:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.655053:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.655056:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655061:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.655066:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.655067:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.655069:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.655071:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116800, 116800] 00000080:00200000:0.0:1788201706.655078:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.655080:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.655100:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.655102:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.655113:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478416896, count: 4096 00000080:00200000:0.0:1788201706.655119:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478416896, bytes: 4096 00000080:00200000:0.0:1788201706.655121:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.655123:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.655124:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478416896, 478420992) 0 0 00000080:00200000:0.0:1788201706.655126:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.655130:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116801, 116801] 00020000:00200000:0.0:1788201706.655131:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478416896 chunk: [478416896, 478420992) 478420992, 4096 00020000:00200000:0.0:1788201706.655133:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.655139:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116801, 116801] 00000008:00010000:0.0:1788201706.655142:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.655144:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.655149:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655154:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478416896 478420991) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655160:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655164:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655168:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.655172:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478416896, 478420992) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.655175:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116801 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.655179:0:17463:0:(rw26.c:898:ll_write_end()) pos 478416896, len 4096, copied 4096 00000080:00200000:0.0:1788201706.655181:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002c56d314[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116801] 00000080:00200000:0.0:1788201706.655182:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fccadb47 fffffc0001029 2:0 ffff93c0c137f380 116801 lru 00000080:00200000:0.0:1788201706.655186:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000064bc002f 116801: 1< 2 - - > 2< 478416896 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68386 - -> 00000080:00200000:0.0:1788201706.655188:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002c56d314 00000080:00200000:0.0:1788201706.655189:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.655190:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.655196:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.655203:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.655205:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.655206:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.655212:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.655214:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.655216:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.655218:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655222:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.655227:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.655228:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.655231:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.655233:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116801, 116801] 00000080:00200000:0.0:1788201706.655239:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.655241:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.655259:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.655261:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.655271:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478420992, count: 4096 00000080:00200000:0.0:1788201706.655277:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478420992, bytes: 4096 00000080:00200000:0.0:1788201706.655279:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.655281:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.655282:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478420992, 478425088) 0 0 00000080:00200000:0.0:1788201706.655284:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.655287:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116802, 116802] 00020000:00200000:0.0:1788201706.655288:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478420992 chunk: [478420992, 478425088) 478425088, 4096 00020000:00200000:0.0:1788201706.655290:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.655296:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116802, 116802] 00000008:00010000:0.0:1788201706.655299:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.655300:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.655305:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655311:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478420992 478425087) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655316:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655321:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655325:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.655330:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478420992, 478425088) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.655332:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116802 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.655336:0:17463:0:(rw26.c:898:ll_write_end()) pos 478420992, len 4096, copied 4096 00000080:00200000:0.0:1788201706.655338:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000111b1c42[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116802] 00000080:00200000:0.0:1788201706.655340:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000041eacfb5 fffffc0001029 2:0 ffff93c0c137f8c0 116802 lru 00000080:00200000:0.0:1788201706.655344:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e4a1af3a 116802: 1< 2 - - > 2< 478420992 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68387 - -> 00000080:00200000:0.0:1788201706.655346:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000111b1c42 00000080:00200000:0.0:1788201706.655346:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.655347:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.655352:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.655360:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.655361:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.655363:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.655368:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.655370:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.655372:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.655374:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655379:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.655383:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.655385:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.655387:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.655389:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116802, 116802] 00000080:00200000:0.0:1788201706.655396:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.655397:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.655415:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.655417:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.655428:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478425088, count: 4096 00000080:00200000:0.0:1788201706.655437:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478425088, bytes: 4096 00000080:00200000:0.0:1788201706.655440:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.655443:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.655445:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478425088, 478429184) 0 0 00000080:00200000:0.0:1788201706.655447:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.655452:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116803, 116803] 00020000:00200000:0.0:1788201706.655455:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478425088 chunk: [478425088, 478429184) 478429184, 4096 00020000:00200000:0.0:1788201706.655457:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.655469:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116803, 116803] 00000008:00010000:0.0:1788201706.655474:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.655476:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.655483:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655490:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478425088 478429183) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655499:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655507:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655514:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.655522:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478425088, 478429184) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.655527:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116803 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.655532:0:17463:0:(rw26.c:898:ll_write_end()) pos 478425088, len 4096, copied 4096 00000080:00200000:0.0:1788201706.655536:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000cb1e0d3b[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116803] 00000080:00200000:0.0:1788201706.655538:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000829fe42b fffffc0001029 2:0 ffff93c0c137fee0 116803 lru 00000080:00200000:0.0:1788201706.655542:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f2caf23b 116803: 1< 2 - - > 2< 478425088 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68388 - -> 00000080:00200000:0.0:1788201706.655543:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000cb1e0d3b 00000080:00200000:0.0:1788201706.655544:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.655546:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.655552:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.655560:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.655562:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.655563:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.655570:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.655571:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.655573:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.655576:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655581:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.655586:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.655587:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.655590:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.655592:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116803, 116803] 00000080:00200000:0.0:1788201706.655599:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.655601:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.655619:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.655621:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.655632:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478429184, count: 4096 00000080:00200000:0.0:1788201706.655638:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478429184, bytes: 4096 00000080:00200000:0.0:1788201706.655639:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.655641:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.655642:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478429184, 478433280) 0 0 00000080:00200000:0.0:1788201706.655644:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.655647:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116804, 116804] 00020000:00200000:0.0:1788201706.655649:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478429184 chunk: [478429184, 478433280) 478433280, 4096 00020000:00200000:0.0:1788201706.655651:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.655657:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116804, 116804] 00000008:00010000:0.0:1788201706.655660:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.655662:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.655667:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655673:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478429184 478433279) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655679:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655683:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655687:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.655691:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478429184, 478433280) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.655695:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116804 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.655698:0:17463:0:(rw26.c:898:ll_write_end()) pos 478429184, len 4096, copied 4096 00000080:00200000:0.0:1788201706.655700:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bd140701[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116804] 00000080:00200000:0.0:1788201706.655702:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009605337f fffffc0001029 2:0 ffff93c0c137f7e0 116804 lru 00000080:00200000:0.0:1788201706.655706:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000024992e78 116804: 1< 2 - - > 2< 478429184 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68389 - -> 00000080:00200000:0.0:1788201706.655707:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bd140701 00000080:00200000:0.0:1788201706.655708:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.655709:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.655715:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.655722:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.655723:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.655724:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.655730:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.655732:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.655733:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.655736:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655741:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.655745:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.655746:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.655749:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.655751:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116804, 116804] 00000080:00200000:0.0:1788201706.655757:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.655759:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.655777:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.655779:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.655789:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478433280, count: 4096 00000080:00200000:0.0:1788201706.655795:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478433280, bytes: 4096 00000080:00200000:0.0:1788201706.655797:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.655799:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.655800:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478433280, 478437376) 0 0 00000080:00200000:0.0:1788201706.655801:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.655804:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116805, 116805] 00020000:00200000:0.0:1788201706.655806:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478433280 chunk: [478433280, 478437376) 478437376, 4096 00020000:00200000:0.0:1788201706.655808:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.655814:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116805, 116805] 00000008:00010000:0.0:1788201706.655817:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.655818:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.655823:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655841:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478433280 478437375) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655848:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655852:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655856:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.655864:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478433280, 478437376) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.655867:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116805 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.655871:0:17463:0:(rw26.c:898:ll_write_end()) pos 478433280, len 4096, copied 4096 00000080:00200000:0.0:1788201706.655873:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a483fffb[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116805] 00000080:00200000:0.0:1788201706.655875:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e836fa05 fffffc0001029 2:0 ffff93c0c137f2a0 116805 lru 00000080:00200000:0.0:1788201706.655879:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005b551b83 116805: 1< 2 - - > 2< 478433280 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68390 - -> 00000080:00200000:0.0:1788201706.655881:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a483fffb 00000080:00200000:0.0:1788201706.655881:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.655883:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.655888:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.655896:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.655898:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.655899:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.655905:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.655906:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.655908:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.655910:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.655915:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.655919:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.655921:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.655924:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.655926:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116805, 116805] 00000080:00200000:0.0:1788201706.655933:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.655934:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.655952:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.655954:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.655964:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478437376, count: 4096 00000080:00200000:0.0:1788201706.655970:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478437376, bytes: 4096 00000080:00200000:0.0:1788201706.655971:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.655974:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.655975:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478437376, 478441472) 0 0 00000080:00200000:0.0:1788201706.655976:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.655979:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116806, 116806] 00020000:00200000:0.0:1788201706.655981:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478437376 chunk: [478437376, 478441472) 478441472, 4096 00020000:00200000:0.0:1788201706.655982:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.655989:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116806, 116806] 00000008:00010000:0.0:1788201706.655992:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.655993:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.655998:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656003:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478437376 478441471) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656009:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656013:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656035:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.656040:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478437376, 478441472) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.656044:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116806 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.656048:0:17463:0:(rw26.c:898:ll_write_end()) pos 478437376, len 4096, copied 4096 00000080:00200000:0.0:1788201706.656050:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000069723195[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116806] 00000080:00200000:0.0:1788201706.656052:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000077f61276 fffffc0001029 2:0 ffff93c0c137fe00 116806 lru 00000080:00200000:0.0:1788201706.656056:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006bea4a11 116806: 1< 2 - - > 2< 478437376 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68391 - -> 00000080:00200000:0.0:1788201706.656057:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000069723195 00000080:00200000:0.0:1788201706.656058:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.656059:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.656065:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.656072:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.656073:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.656075:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.656081:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.656083:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.656084:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.656087:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656092:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.656100:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.656102:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.656104:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.656106:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116806, 116806] 00000080:00200000:0.0:1788201706.656113:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.656115:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.656133:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.656135:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.656145:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478441472, count: 4096 00000080:00200000:0.0:1788201706.656151:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478441472, bytes: 4096 00000080:00200000:0.0:1788201706.656153:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.656155:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.656156:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478441472, 478445568) 0 0 00000080:00200000:0.0:1788201706.656157:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.656161:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116807, 116807] 00020000:00200000:0.0:1788201706.656162:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478441472 chunk: [478441472, 478445568) 478445568, 4096 00020000:00200000:0.0:1788201706.656164:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.656170:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116807, 116807] 00000008:00010000:0.0:1788201706.656173:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.656175:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.656180:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656185:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478441472 478445567) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656191:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656199:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656203:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.656207:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478441472, 478445568) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.656210:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116807 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.656214:0:17463:0:(rw26.c:898:ll_write_end()) pos 478441472, len 4096, copied 4096 00000080:00200000:0.0:1788201706.656216:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000eab36fe0[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116807] 00000080:00200000:0.0:1788201706.656217:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000804cb5df fffffc0001029 2:0 ffff93c0c137f1c0 116807 lru 00000080:00200000:0.0:1788201706.656221:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000082d8b2c6 116807: 1< 2 - - > 2< 478441472 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68392 - -> 00000080:00200000:0.0:1788201706.656222:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000eab36fe0 00000080:00200000:0.0:1788201706.656222:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.656224:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.656229:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.656237:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.656238:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.656240:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.656245:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.656247:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.656248:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.656251:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656258:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.656263:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.656264:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.656267:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.656269:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116807, 116807] 00000080:00200000:0.0:1788201706.656275:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.656277:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.656294:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.656296:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.656306:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478445568, count: 4096 00000080:00200000:0.0:1788201706.656312:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478445568, bytes: 4096 00000080:00200000:0.0:1788201706.656314:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.656316:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.656317:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478445568, 478449664) 0 0 00000080:00200000:0.0:1788201706.656319:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.656321:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116808, 116808] 00020000:00200000:0.0:1788201706.656323:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478445568 chunk: [478445568, 478449664) 478449664, 4096 00020000:00200000:0.0:1788201706.656325:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.656331:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116808, 116808] 00000008:00010000:0.0:1788201706.656334:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.656335:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.656340:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656346:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478445568 478449663) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656354:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656359:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656363:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.656367:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478445568, 478449664) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.656370:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116808 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.656373:0:17463:0:(rw26.c:898:ll_write_end()) pos 478445568, len 4096, copied 4096 00000080:00200000:0.0:1788201706.656375:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005bf6255d[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116808] 00000080:00200000:0.0:1788201706.656377:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004b229515 fffffc0001029 2:0 ffff93c0c137f620 116808 lru 00000080:00200000:0.0:1788201706.656380:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004fb8dd94 116808: 1< 2 - - > 2< 478445568 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68393 - -> 00000080:00200000:0.0:1788201706.656381:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005bf6255d 00000080:00200000:0.0:1788201706.656382:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.656383:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.656389:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.656396:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.656398:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.656399:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.656405:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.656406:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.656408:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.656411:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656416:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.656420:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.656422:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.656424:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.656427:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116808, 116808] 00000080:00200000:0.0:1788201706.656436:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.656440:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.656467:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.656471:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.656486:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478449664, count: 4096 00000080:00200000:0.0:1788201706.656497:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478449664, bytes: 4096 00000080:00200000:0.0:1788201706.656500:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.656503:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.656505:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478449664, 478453760) 0 0 00000080:00200000:0.0:1788201706.656507:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.656512:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116809, 116809] 00020000:00200000:0.0:1788201706.656515:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478449664 chunk: [478449664, 478453760) 478453760, 4096 00020000:00200000:0.0:1788201706.656517:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.656529:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116809, 116809] 00000008:00010000:0.0:1788201706.656534:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.656536:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.656544:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656558:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478449664 478453759) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656564:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656569:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656573:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.656578:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478449664, 478453760) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.656581:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116809 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.656585:0:17463:0:(rw26.c:898:ll_write_end()) pos 478449664, len 4096, copied 4096 00000080:00200000:0.0:1788201706.656587:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002e7d8137[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116809] 00000080:00200000:0.0:1788201706.656589:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a986860d fffffc0001029 2:0 ffff93c0c137fd20 116809 lru 00000080:00200000:0.0:1788201706.656592:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002f0809ba 116809: 1< 2 - - > 2< 478449664 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68394 - -> 00000080:00200000:0.0:1788201706.656593:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002e7d8137 00000080:00200000:0.0:1788201706.656594:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.656595:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.656601:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.656609:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.656611:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.656613:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.656619:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.656621:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.656623:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.656625:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656630:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.656635:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.656637:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.656639:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.656641:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116809, 116809] 00000080:00200000:0.0:1788201706.656648:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.656650:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.656668:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.656671:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.656681:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478453760, count: 4096 00000080:00200000:0.0:1788201706.656687:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478453760, bytes: 4096 00000080:00200000:0.0:1788201706.656689:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.656691:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.656692:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478453760, 478457856) 0 0 00000080:00200000:0.0:1788201706.656693:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.656697:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116810, 116810] 00020000:00200000:0.0:1788201706.656698:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478453760 chunk: [478453760, 478457856) 478457856, 4096 00020000:00200000:0.0:1788201706.656700:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.656706:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116810, 116810] 00000008:00010000:0.0:1788201706.656709:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.656711:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.656716:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656722:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478453760 478457855) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656727:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656731:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656735:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.656739:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478453760, 478457856) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.656742:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116810 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.656745:0:17463:0:(rw26.c:898:ll_write_end()) pos 478453760, len 4096, copied 4096 00000080:00200000:0.0:1788201706.656747:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d8f32080[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116810] 00000080:00200000:0.0:1788201706.656749:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008c7065e8 fffffc0001029 2:0 ffff93c0c137f9a0 116810 lru 00000080:00200000:0.0:1788201706.656753:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a52b9bba 116810: 1< 2 - - > 2< 478453760 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68395 - -> 00000080:00200000:0.0:1788201706.656754:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d8f32080 00000080:00200000:0.0:1788201706.656755:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.656756:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.656762:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.656769:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.656771:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.656772:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.656778:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.656780:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.656781:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.656783:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656788:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.656792:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.656794:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.656796:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.656798:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116810, 116810] 00000080:00200000:0.0:1788201706.656805:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.656807:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.656824:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.656826:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.656850:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478457856, count: 4096 00000080:00200000:0.0:1788201706.656857:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478457856, bytes: 4096 00000080:00200000:0.0:1788201706.656859:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.656861:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.656862:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478457856, 478461952) 0 0 00000080:00200000:0.0:1788201706.656863:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.656867:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116811, 116811] 00020000:00200000:0.0:1788201706.656868:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478457856 chunk: [478457856, 478461952) 478461952, 4096 00020000:00200000:0.0:1788201706.656870:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.656876:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116811, 116811] 00000008:00010000:0.0:1788201706.656880:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.656881:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.656886:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656891:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478457856 478461951) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656897:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656901:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656906:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.656910:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478457856, 478461952) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.656912:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116811 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.656916:0:17463:0:(rw26.c:898:ll_write_end()) pos 478457856, len 4096, copied 4096 00000080:00200000:0.0:1788201706.656918:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c833385f[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116811] 00000080:00200000:0.0:1788201706.656920:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000062cc5fd9 fffffc0001029 2:0 ffff93c0c137fb60 116811 lru 00000080:00200000:0.0:1788201706.656923:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000061cef42c 116811: 1< 2 - - > 2< 478457856 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68396 - -> 00000080:00200000:0.0:1788201706.656924:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c833385f 00000080:00200000:0.0:1788201706.656925:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.656926:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.656931:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.656942:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.656944:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.656945:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.656951:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.656953:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.656954:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.656957:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.656961:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.656967:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.656968:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.656971:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.656972:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116811, 116811] 00000080:00200000:0.0:1788201706.656979:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.656981:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.656999:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.657002:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.657027:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478461952, count: 4096 00000080:00200000:0.0:1788201706.657039:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478461952, bytes: 4096 00000080:00200000:0.0:1788201706.657042:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.657045:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.657047:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478461952, 478466048) 0 0 00000080:00200000:0.0:1788201706.657049:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.657054:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116812, 116812] 00020000:00200000:0.0:1788201706.657058:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478461952 chunk: [478461952, 478466048) 478466048, 4096 00020000:00200000:0.0:1788201706.657060:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.657069:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116812, 116812] 00000008:00010000:0.0:1788201706.657072:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.657074:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.657079:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657085:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478461952 478466047) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657090:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657095:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657099:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.657104:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478461952, 478466048) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.657107:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116812 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.657110:0:17463:0:(rw26.c:898:ll_write_end()) pos 478461952, len 4096, copied 4096 00000080:00200000:0.0:1788201706.657113:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000c064f4a[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116812] 00000080:00200000:0.0:1788201706.657115:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000074c4893 fffffc0001029 2:0 ffff93c0c137fc40 116812 lru 00000080:00200000:0.0:1788201706.657119:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000083d44c46 116812: 1< 2 - - > 2< 478461952 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68397 - -> 00000080:00200000:0.0:1788201706.657120:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000c064f4a 00000080:00200000:0.0:1788201706.657121:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.657122:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.657129:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.657136:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.657138:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.657139:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.657146:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.657147:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.657149:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.657151:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657156:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.657161:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.657162:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.657165:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.657167:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116812, 116812] 00000080:00200000:0.0:1788201706.657174:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.657176:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.657203:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.657207:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.657222:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478466048, count: 4096 00000080:00200000:0.0:1788201706.657232:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478466048, bytes: 4096 00000080:00200000:0.0:1788201706.657235:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.657239:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.657241:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478466048, 478470144) 0 0 00000080:00200000:0.0:1788201706.657244:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.657249:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116813, 116813] 00020000:00200000:0.0:1788201706.657252:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478466048 chunk: [478466048, 478470144) 478470144, 4096 00020000:00200000:0.0:1788201706.657254:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.657266:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116813, 116813] 00000008:00010000:0.0:1788201706.657272:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.657274:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.657282:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657291:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478466048 478470143) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657301:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657309:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657316:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.657323:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478466048, 478470144) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.657327:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116813 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.657332:0:17463:0:(rw26.c:898:ll_write_end()) pos 478466048, len 4096, copied 4096 00000080:00200000:0.0:1788201706.657336:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000550ae8ad[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116813] 00000080:00200000:0.0:1788201706.657338:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000059c3b51f fffffc0001029 2:0 ffff93c0c137f540 116813 lru 00000080:00200000:0.0:1788201706.657342:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008bf59dbf 116813: 1< 2 - - > 2< 478466048 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68398 - -> 00000080:00200000:0.0:1788201706.657344:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000550ae8ad 00000080:00200000:0.0:1788201706.657345:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.657347:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.657355:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.657367:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.657371:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.657373:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.657382:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.657385:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.657387:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.657392:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657400:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.657407:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.657410:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.657414:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.657418:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116813, 116813] 00000080:00200000:0.0:1788201706.657429:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.657432:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.657461:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.657465:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.657481:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478470144, count: 4096 00000080:00200000:0.0:1788201706.657492:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478470144, bytes: 4096 00000080:00200000:0.0:1788201706.657495:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.657499:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.657500:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478470144, 478474240) 0 0 00000080:00200000:0.0:1788201706.657502:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.657507:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116814, 116814] 00020000:00200000:0.0:1788201706.657510:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478470144 chunk: [478470144, 478474240) 478474240, 4096 00020000:00200000:0.0:1788201706.657512:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.657524:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116814, 116814] 00000008:00010000:0.0:1788201706.657530:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.657532:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.657540:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657548:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478470144 478474239) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657557:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657565:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657572:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.657579:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478470144, 478474240) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.657585:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116814 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.657590:0:17463:0:(rw26.c:898:ll_write_end()) pos 478470144, len 4096, copied 4096 00000080:00200000:0.0:1788201706.657594:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000323cfc4[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116814] 00000080:00200000:0.0:1788201706.657597:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000edae4f60 fffffc0001029 2:0 ffff93c0c137f460 116814 lru 00000080:00200000:0.0:1788201706.657603:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000415abf1 116814: 1< 2 - - > 2< 478470144 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68399 - -> 00000080:00200000:0.0:1788201706.657606:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000323cfc4 00000080:00200000:0.0:1788201706.657607:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.657609:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.657617:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.657629:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.657632:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.657635:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.657644:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.657646:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.657649:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.657653:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657660:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.657670:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.657672:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.657676:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.657680:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116814, 116814] 00000080:00200000:0.0:1788201706.657691:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.657694:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.657722:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.657727:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.657743:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478474240, count: 4096 00000080:00200000:0.0:1788201706.657754:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478474240, bytes: 4096 00000080:00200000:0.0:1788201706.657757:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.657760:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.657762:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478474240, 478478336) 0 0 00000080:00200000:0.0:1788201706.657764:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.657770:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116815, 116815] 00020000:00200000:0.0:1788201706.657773:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478474240 chunk: [478474240, 478478336) 478478336, 4096 00020000:00200000:0.0:1788201706.657775:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.657787:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116815, 116815] 00000008:00010000:0.0:1788201706.657792:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.657795:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.657802:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657812:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478474240 478478335) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657821:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657845:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657854:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.657862:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478474240, 478478336) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.657867:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116815 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.657873:0:17463:0:(rw26.c:898:ll_write_end()) pos 478474240, len 4096, copied 4096 00000080:00200000:0.0:1788201706.657878:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f5c50da4[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116815] 00000080:00200000:0.0:1788201706.657881:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000de114ed3 fffffc0001029 2:0 ffff93c0c13807e0 116815 lru 00000080:00200000:0.0:1788201706.657886:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e643cfd3 116815: 1< 2 - - > 2< 478474240 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68400 - -> 00000080:00200000:0.0:1788201706.657888:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f5c50da4 00000080:00200000:0.0:1788201706.657889:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.657891:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.657900:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.657912:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.657915:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.657917:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.657928:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.657931:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.657934:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.657938:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.657947:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.657956:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.657958:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.657962:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.657966:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116815, 116815] 00000080:00200000:0.0:1788201706.657977:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.657980:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.658011:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.658029:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.658047:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478478336, count: 4096 00000080:00200000:0.0:1788201706.658059:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478478336, bytes: 4096 00000080:00200000:0.0:1788201706.658063:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.658066:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.658068:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478478336, 478482432) 0 0 00000080:00200000:0.0:1788201706.658070:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.658075:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116816, 116816] 00020000:00200000:0.0:1788201706.658079:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478478336 chunk: [478478336, 478482432) 478482432, 4096 00020000:00200000:0.0:1788201706.658082:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.658094:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116816, 116816] 00000008:00010000:0.0:1788201706.658099:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.658101:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.658109:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658118:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478478336 478482431) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658127:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658135:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658142:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.658150:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478478336, 478482432) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.658156:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116816 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.658162:0:17463:0:(rw26.c:898:ll_write_end()) pos 478478336, len 4096, copied 4096 00000080:00200000:0.0:1788201706.658166:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000077efd48a[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116816] 00000080:00200000:0.0:1788201706.658169:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004217ea9b fffffc0001029 2:0 ffff93c0c13802a0 116816 lru 00000080:00200000:0.0:1788201706.658175:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000744f2bb4 116816: 1< 2 - - > 2< 478478336 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68401 - -> 00000080:00200000:0.0:1788201706.658177:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000077efd48a 00000080:00200000:0.0:1788201706.658178:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.658180:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.658189:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.658201:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.658204:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.658207:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.658217:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.658219:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.658222:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.658226:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658234:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.658242:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.658245:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.658250:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.658253:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116816, 116816] 00000080:00200000:0.0:1788201706.658264:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.658267:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.658299:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.658302:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.658319:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478482432, count: 4096 00000080:00200000:0.0:1788201706.658331:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478482432, bytes: 4096 00000080:00200000:0.0:1788201706.658334:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.658338:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.658339:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478482432, 478486528) 0 0 00000080:00200000:0.0:1788201706.658342:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.658347:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116817, 116817] 00020000:00200000:0.0:1788201706.658350:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478482432 chunk: [478482432, 478486528) 478486528, 4096 00020000:00200000:0.0:1788201706.658352:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.658365:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116817, 116817] 00000008:00010000:0.0:1788201706.658371:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.658373:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.658381:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658390:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478482432 478486527) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658399:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658407:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658414:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.658428:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478482432, 478486528) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.658433:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116817 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.658439:0:17463:0:(rw26.c:898:ll_write_end()) pos 478482432, len 4096, copied 4096 00000080:00200000:0.0:1788201706.658443:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008c5d7986[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116817] 00000080:00200000:0.0:1788201706.658446:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000564acccb fffffc0001029 2:0 ffff93c0c1380e00 116817 lru 00000080:00200000:0.0:1788201706.658452:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d06adc95 116817: 1< 2 - - > 2< 478482432 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68402 - -> 00000080:00200000:0.0:1788201706.658455:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008c5d7986 00000080:00200000:0.0:1788201706.658455:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.658458:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.658467:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.658479:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.658482:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.658485:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.658494:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.658497:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.658500:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.658504:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658512:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.658521:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.658523:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.658528:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.658532:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116817, 116817] 00000080:00200000:0.0:1788201706.658543:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.658546:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.658576:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.658579:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.658595:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478486528, count: 4096 00000080:00200000:0.0:1788201706.658606:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478486528, bytes: 4096 00000080:00200000:0.0:1788201706.658610:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.658613:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.658615:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478486528, 478490624) 0 0 00000080:00200000:0.0:1788201706.658617:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.658622:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116818, 116818] 00020000:00200000:0.0:1788201706.658625:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478486528 chunk: [478486528, 478490624) 478490624, 4096 00020000:00200000:0.0:1788201706.658628:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.658640:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116818, 116818] 00000008:00010000:0.0:1788201706.658645:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.658648:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.658656:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658664:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478486528 478490623) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658673:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658680:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658692:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.658700:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478486528, 478490624) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.658705:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116818 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.658710:0:17463:0:(rw26.c:898:ll_write_end()) pos 478486528, len 4096, copied 4096 00000080:00200000:0.0:1788201706.658714:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000041f90996[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116818] 00000080:00200000:0.0:1788201706.658717:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b6a3e250 fffffc0001029 2:0 ffff93c0c13801c0 116818 lru 00000080:00200000:0.0:1788201706.658723:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006d29aa7d 116818: 1< 2 - - > 2< 478486528 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68403 - -> 00000080:00200000:0.0:1788201706.658726:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000041f90996 00000080:00200000:0.0:1788201706.658727:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.658728:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.658737:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.658749:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.658752:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.658754:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.658763:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.658766:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.658768:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.658772:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658781:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.658794:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.658797:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.658801:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.658804:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116818, 116818] 00000080:00200000:0.0:1788201706.658814:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.658817:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.658871:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.658875:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.658894:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478490624, count: 4096 00000080:00200000:0.0:1788201706.658905:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478490624, bytes: 4096 00000080:00200000:0.0:1788201706.658909:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.658913:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.658914:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478490624, 478494720) 0 0 00000080:00200000:0.0:1788201706.658917:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.658923:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116819, 116819] 00020000:00200000:0.0:1788201706.658926:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478490624 chunk: [478490624, 478494720) 478494720, 4096 00020000:00200000:0.0:1788201706.658928:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.658940:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116819, 116819] 00000008:00010000:0.0:1788201706.658946:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.658948:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.658956:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658965:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478490624 478494719) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658974:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658988:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.658995:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.659002:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478490624, 478494720) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.659007:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116819 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.659013:0:17463:0:(rw26.c:898:ll_write_end()) pos 478490624, len 4096, copied 4096 00000080:00200000:0.0:1788201706.659032:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b66eeaa4[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116819] 00000080:00200000:0.0:1788201706.659035:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008878e9d4 fffffc0001029 2:0 ffff93c0c1380620 116819 lru 00000080:00200000:0.0:1788201706.659040:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cb265ba1 116819: 1< 2 - - > 2< 478490624 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68404 - -> 00000080:00200000:0.0:1788201706.659043:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b66eeaa4 00000080:00200000:0.0:1788201706.659044:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.659046:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.659054:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.659067:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.659070:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.659072:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.659082:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.659085:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.659087:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.659091:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659104:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.659113:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.659115:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.659120:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.659123:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116819, 116819] 00000080:00200000:0.0:1788201706.659134:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.659137:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.659167:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.659171:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.659187:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478494720, count: 4096 00000080:00200000:0.0:1788201706.659199:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478494720, bytes: 4096 00000080:00200000:0.0:1788201706.659202:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.659205:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.659207:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478494720, 478498816) 0 0 00000080:00200000:0.0:1788201706.659209:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.659214:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116820, 116820] 00020000:00200000:0.0:1788201706.659218:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478494720 chunk: [478494720, 478498816) 478498816, 4096 00020000:00200000:0.0:1788201706.659220:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.659232:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116820, 116820] 00000008:00010000:0.0:1788201706.659238:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.659240:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.659247:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659257:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478494720 478498815) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659270:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659278:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659285:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.659292:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478494720, 478498816) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.659298:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116820 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.659303:0:17463:0:(rw26.c:898:ll_write_end()) pos 478494720, len 4096, copied 4096 00000080:00200000:0.0:1788201706.659307:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000073daa46f[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116820] 00000080:00200000:0.0:1788201706.659310:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000eb1f5199 fffffc0001029 2:0 ffff93c0c1380d20 116820 lru 00000080:00200000:0.0:1788201706.659316:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d18c683b 116820: 1< 2 - - > 2< 478494720 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68405 - -> 00000080:00200000:0.0:1788201706.659319:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000073daa46f 00000080:00200000:0.0:1788201706.659320:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.659322:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.659330:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.659343:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.659346:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.659348:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.659357:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.659360:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.659363:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.659367:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659376:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.659384:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.659386:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.659390:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.659393:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116820, 116820] 00000080:00200000:0.0:1788201706.659403:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.659406:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.659432:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.659436:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.659450:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478498816, count: 4096 00000080:00200000:0.0:1788201706.659460:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478498816, bytes: 4096 00000080:00200000:0.0:1788201706.659463:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.659466:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.659468:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478498816, 478502912) 0 0 00000080:00200000:0.0:1788201706.659470:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.659475:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116821, 116821] 00020000:00200000:0.0:1788201706.659478:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478498816 chunk: [478498816, 478502912) 478502912, 4096 00020000:00200000:0.0:1788201706.659480:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.659492:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116821, 116821] 00000008:00010000:0.0:1788201706.659496:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.659498:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.659506:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659520:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478498816 478502911) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659529:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659536:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659544:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.659552:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478498816, 478502912) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.659557:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116821 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.659562:0:17463:0:(rw26.c:898:ll_write_end()) pos 478498816, len 4096, copied 4096 00000080:00200000:0.0:1788201706.659566:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ac0c67cf[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116821] 00000080:00200000:0.0:1788201706.659569:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000fe26de2 fffffc0001029 2:0 ffff93c0c13809a0 116821 lru 00000080:00200000:0.0:1788201706.659575:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c206abac 116821: 1< 2 - - > 2< 478498816 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68406 - -> 00000080:00200000:0.0:1788201706.659577:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ac0c67cf 00000080:00200000:0.0:1788201706.659578:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.659580:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.659588:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.659601:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.659604:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.659606:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.659615:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.659618:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.659621:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.659625:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659633:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.659641:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.659643:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.659648:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.659651:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116821, 116821] 00000080:00200000:0.0:1788201706.659662:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.659666:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.659694:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.659698:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.659714:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478502912, count: 4096 00000080:00200000:0.0:1788201706.659725:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478502912, bytes: 4096 00000080:00200000:0.0:1788201706.659728:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.659732:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.659733:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478502912, 478507008) 0 0 00000080:00200000:0.0:1788201706.659736:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.659741:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116822, 116822] 00020000:00200000:0.0:1788201706.659744:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478502912 chunk: [478502912, 478507008) 478507008, 4096 00020000:00200000:0.0:1788201706.659747:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.659758:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116822, 116822] 00000008:00010000:0.0:1788201706.659764:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.659766:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.659774:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659783:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478502912 478507007) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659792:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659799:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659807:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.659814:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478502912, 478507008) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.659819:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116822 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.659825:0:17463:0:(rw26.c:898:ll_write_end()) pos 478502912, len 4096, copied 4096 00000080:00200000:0.0:1788201706.659844:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000010eb0fb7[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116822] 00000080:00200000:0.0:1788201706.659848:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000041802657 fffffc0001029 2:0 ffff93c0c1380b60 116822 lru 00000080:00200000:0.0:1788201706.659854:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008441045f 116822: 1< 2 - - > 2< 478502912 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68407 - -> 00000080:00200000:0.0:1788201706.659856:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000010eb0fb7 00000080:00200000:0.0:1788201706.659857:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.659859:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.659868:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.659880:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.659884:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.659886:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.659895:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.659898:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.659900:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.659905:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.659913:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.659921:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.659924:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.659928:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.659931:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116822, 116822] 00000080:00200000:0.0:1788201706.659941:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.659945:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.659975:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.659978:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.659995:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478507008, count: 4096 00000080:00200000:0.0:1788201706.660006:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478507008, bytes: 4096 00000080:00200000:0.0:1788201706.660009:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.660013:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.660031:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478507008, 478511104) 0 0 00000080:00200000:0.0:1788201706.660033:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.660039:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116823, 116823] 00020000:00200000:0.0:1788201706.660042:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478507008 chunk: [478507008, 478511104) 478511104, 4096 00020000:00200000:0.0:1788201706.660044:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.660057:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116823, 116823] 00000008:00010000:0.0:1788201706.660063:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.660066:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.660073:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660082:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478507008 478511103) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660091:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660100:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660107:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.660115:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478507008, 478511104) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.660120:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116823 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.660126:0:17463:0:(rw26.c:898:ll_write_end()) pos 478507008, len 4096, copied 4096 00000080:00200000:0.0:1788201706.660130:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000006b9ae1a[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116823] 00000080:00200000:0.0:1788201706.660133:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008e6952d2 fffffc0001029 2:0 ffff93c0c1380c40 116823 lru 00000080:00200000:0.0:1788201706.660139:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000428b9411 116823: 1< 2 - - > 2< 478507008 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68408 - -> 00000080:00200000:0.0:1788201706.660141:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000006b9ae1a 00000080:00200000:0.0:1788201706.660143:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.660145:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.660153:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.660169:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.660173:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.660175:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.660185:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.660187:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.660190:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.660195:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660203:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.660211:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.660214:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.660218:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.660221:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116823, 116823] 00000080:00200000:0.0:1788201706.660233:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.660236:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.660267:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.660271:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.660287:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478511104, count: 4096 00000080:00200000:0.0:1788201706.660298:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478511104, bytes: 4096 00000080:00200000:0.0:1788201706.660301:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.660305:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.660307:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478511104, 478515200) 0 0 00000080:00200000:0.0:1788201706.660309:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.660314:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116824, 116824] 00020000:00200000:0.0:1788201706.660317:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478511104 chunk: [478511104, 478515200) 478515200, 4096 00020000:00200000:0.0:1788201706.660320:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.660332:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116824, 116824] 00000008:00010000:0.0:1788201706.660338:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.660341:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.660348:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660357:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478511104 478515199) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660366:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660374:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660381:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.660388:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478511104, 478515200) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.660393:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116824 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.660399:0:17463:0:(rw26.c:898:ll_write_end()) pos 478511104, len 4096, copied 4096 00000080:00200000:0.0:1788201706.660403:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000004353c1b[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116824] 00000080:00200000:0.0:1788201706.660406:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000071ab4d9b fffffc0001029 2:0 ffff93c0c1380540 116824 lru 00000080:00200000:0.0:1788201706.660412:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ce2d2f89 116824: 1< 2 - - > 2< 478511104 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68409 - -> 00000080:00200000:0.0:1788201706.660414:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000004353c1b 00000080:00200000:0.0:1788201706.660415:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.660418:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.660426:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.660439:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.660442:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.660444:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.660453:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.660456:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.660459:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.660463:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660471:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.660479:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.660482:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.660486:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.660490:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116824, 116824] 00000080:00200000:0.0:1788201706.660501:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.660504:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.660533:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.660536:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.660553:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478515200, count: 4096 00000080:00200000:0.0:1788201706.660564:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478515200, bytes: 4096 00000080:00200000:0.0:1788201706.660566:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.660570:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.660572:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478515200, 478519296) 0 0 00000080:00200000:0.0:1788201706.660575:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.660581:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116825, 116825] 00020000:00200000:0.0:1788201706.660584:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478515200 chunk: [478515200, 478519296) 478519296, 4096 00020000:00200000:0.0:1788201706.660586:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.660599:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116825, 116825] 00000008:00010000:0.0:1788201706.660605:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.660607:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.660614:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660623:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478515200 478519295) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660632:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660639:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660647:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.660655:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478515200, 478519296) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.660660:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116825 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.660666:0:17463:0:(rw26.c:898:ll_write_end()) pos 478515200, len 4096, copied 4096 00000080:00200000:0.0:1788201706.660670:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000059c2f83c[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116825] 00000080:00200000:0.0:1788201706.660673:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000d127d27 fffffc0001029 2:0 ffff93c0c1380460 116825 lru 00000080:00200000:0.0:1788201706.660679:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005198ae5e 116825: 1< 2 - - > 2< 478515200 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68410 - -> 00000080:00200000:0.0:1788201706.660683:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000059c2f83c 00000080:00200000:0.0:1788201706.660684:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.660686:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.660694:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.660705:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.660708:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.660710:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.660719:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.660722:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.660724:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.660728:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660736:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.660744:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.660747:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.660751:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.660753:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116825, 116825] 00000080:00200000:0.0:1788201706.660764:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.660767:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.660794:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.660797:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.660812:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478519296, count: 4096 00000080:00200000:0.0:1788201706.660823:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478519296, bytes: 4096 00000080:00200000:0.0:1788201706.660826:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.660845:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.660847:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478519296, 478523392) 0 0 00000080:00200000:0.0:1788201706.660849:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.660853:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116826, 116826] 00020000:00200000:0.0:1788201706.660856:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478519296 chunk: [478519296, 478523392) 478523392, 4096 00020000:00200000:0.0:1788201706.660858:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.660869:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116826, 116826] 00000008:00010000:0.0:1788201706.660874:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.660876:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.660883:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660891:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478519296 478523391) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660901:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660908:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660915:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.660922:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478519296, 478523392) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.660926:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116826 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.660931:0:17463:0:(rw26.c:898:ll_write_end()) pos 478519296, len 4096, copied 4096 00000080:00200000:0.0:1788201706.660934:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f884d393[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116826] 00000080:00200000:0.0:1788201706.660938:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b5a78b9e fffffc0001029 2:0 ffff93c0c13800e0 116826 lru 00000080:00200000:0.0:1788201706.660942:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000db71f808 116826: 1< 2 - - > 2< 478519296 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68411 - -> 00000080:00200000:0.0:1788201706.660944:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f884d393 00000080:00200000:0.0:1788201706.660945:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.660947:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.660954:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.660964:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.660967:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.660969:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.660976:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.660979:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.660981:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.660985:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.660991:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.660998:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.661001:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.661005:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.661007:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116826, 116826] 00000080:00200000:0.0:1788201706.661033:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.661037:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.661066:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.661069:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.661085:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478523392, count: 4096 00000080:00200000:0.0:1788201706.661096:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478523392, bytes: 4096 00000080:00200000:0.0:1788201706.661099:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.661103:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.661105:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478523392, 478527488) 0 0 00000080:00200000:0.0:1788201706.661107:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.661112:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116827, 116827] 00020000:00200000:0.0:1788201706.661115:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478523392 chunk: [478523392, 478527488) 478527488, 4096 00020000:00200000:0.0:1788201706.661117:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.661128:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116827, 116827] 00000008:00010000:0.0:1788201706.661132:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.661134:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.661142:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661151:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478523392 478527487) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661160:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661168:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661176:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.661183:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478523392, 478527488) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.661187:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116827 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.661192:0:17463:0:(rw26.c:898:ll_write_end()) pos 478523392, len 4096, copied 4096 00000080:00200000:0.0:1788201706.661196:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000024c425b9[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116827] 00000080:00200000:0.0:1788201706.661198:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006a4254ab fffffc0001029 2:0 ffff93c0c1380700 116827 lru 00000080:00200000:0.0:1788201706.661204:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006ba5c64c 116827: 1< 2 - - > 2< 478523392 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68412 - -> 00000080:00200000:0.0:1788201706.661206:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000024c425b9 00000080:00200000:0.0:1788201706.661207:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.661209:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.661218:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.661230:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.661233:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.661236:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.661246:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.661249:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.661251:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.661256:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661264:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.661272:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.661275:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.661278:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.661281:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116827, 116827] 00000080:00200000:0.0:1788201706.661291:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.661294:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.661319:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.661323:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.661337:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478527488, count: 4096 00000080:00200000:0.0:1788201706.661347:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478527488, bytes: 4096 00000080:00200000:0.0:1788201706.661349:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.661353:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.661354:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478527488, 478531584) 0 0 00000080:00200000:0.0:1788201706.661356:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.661361:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116828, 116828] 00020000:00200000:0.0:1788201706.661364:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478527488 chunk: [478527488, 478531584) 478531584, 4096 00020000:00200000:0.0:1788201706.661366:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.661376:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116828, 116828] 00000008:00010000:0.0:1788201706.661381:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.661383:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.661390:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661399:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478527488 478531583) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661408:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661416:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661420:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.661425:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478527488, 478531584) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.661429:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116828 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.661432:0:17463:0:(rw26.c:898:ll_write_end()) pos 478527488, len 4096, copied 4096 00000080:00200000:0.0:1788201706.661435:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006432cc53[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116828] 00000080:00200000:0.0:1788201706.661437:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000067ea95b7 fffffc0001029 2:0 ffff93c0c1380000 116828 lru 00000080:00200000:0.0:1788201706.661441:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d0eac186 116828: 1< 2 - - > 2< 478527488 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68413 - -> 00000080:00200000:0.0:1788201706.661442:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006432cc53 00000080:00200000:0.0:1788201706.661443:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.661444:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.661450:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.661458:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.661459:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.661461:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.661467:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.661469:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.661470:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.661472:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661477:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.661482:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.661483:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.661486:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.661488:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116828, 116828] 00000080:00200000:0.0:1788201706.661495:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.661497:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.661515:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.661517:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.661527:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478531584, count: 4096 00000080:00200000:0.0:1788201706.661533:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478531584, bytes: 4096 00000080:00200000:0.0:1788201706.661535:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.661537:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.661538:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478531584, 478535680) 0 0 00000080:00200000:0.0:1788201706.661540:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.661543:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116829, 116829] 00020000:00200000:0.0:1788201706.661545:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478531584 chunk: [478531584, 478535680) 478535680, 4096 00020000:00200000:0.0:1788201706.661547:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.661553:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116829, 116829] 00000008:00010000:0.0:1788201706.661556:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.661557:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.661562:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661568:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478531584 478535679) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661574:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661578:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661582:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.661590:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478531584, 478535680) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.661593:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116829 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.661597:0:17463:0:(rw26.c:898:ll_write_end()) pos 478531584, len 4096, copied 4096 00000080:00200000:0.0:1788201706.661599:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a7aae7fa[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116829] 00000080:00200000:0.0:1788201706.661601:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003c70bf80 fffffc0001029 2:0 ffff93c0c1380a80 116829 lru 00000080:00200000:0.0:1788201706.661604:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008bc463f1 116829: 1< 2 - - > 2< 478531584 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68414 - -> 00000080:00200000:0.0:1788201706.661606:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a7aae7fa 00000080:00200000:0.0:1788201706.661607:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.661608:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.661613:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.661620:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.661622:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.661623:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.661629:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.661631:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.661632:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.661635:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661640:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.661645:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.661646:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.661649:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.661651:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116829, 116829] 00000080:00200000:0.0:1788201706.661657:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.661659:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.661677:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.661679:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.661689:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478535680, count: 4096 00000080:00200000:0.0:1788201706.661695:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478535680, bytes: 4096 00000080:00200000:0.0:1788201706.661697:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.661699:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.661700:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478535680, 478539776) 0 0 00000080:00200000:0.0:1788201706.661701:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.661704:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116830, 116830] 00020000:00200000:0.0:1788201706.661706:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478535680 chunk: [478535680, 478539776) 478539776, 4096 00020000:00200000:0.0:1788201706.661708:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.661714:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116830, 116830] 00000008:00010000:0.0:1788201706.661717:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.661718:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.661723:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661729:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478535680 478539775) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661734:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661739:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661746:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.661750:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478535680, 478539776) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.661753:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116830 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.661756:0:17463:0:(rw26.c:898:ll_write_end()) pos 478535680, len 4096, copied 4096 00000080:00200000:0.0:1788201706.661759:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fc51c0d1[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116830] 00000080:00200000:0.0:1788201706.661760:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000061067897 fffffc0001029 2:0 ffff93c0c1380380 116830 lru 00000080:00200000:0.0:1788201706.661764:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007670f6bf 116830: 1< 2 - - > 2< 478535680 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68415 - -> 00000080:00200000:0.0:1788201706.661765:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fc51c0d1 00000080:00200000:0.0:1788201706.661766:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.661767:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.661773:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.661780:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.661782:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.661783:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.661789:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.661791:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.661792:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.661795:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661799:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.661808:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.661809:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.661812:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.661814:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116830, 116830] 00000080:00200000:0.0:1788201706.661820:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.661822:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.661862:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.661865:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.661876:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478539776, count: 4096 00000080:00200000:0.0:1788201706.661883:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478539776, bytes: 4096 00000080:00200000:0.0:1788201706.661885:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.661887:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.661888:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478539776, 478543872) 0 0 00000080:00200000:0.0:1788201706.661889:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.661893:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116831, 116831] 00020000:00200000:0.0:1788201706.661895:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478539776 chunk: [478539776, 478543872) 478543872, 4096 00020000:00200000:0.0:1788201706.661896:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.661903:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116831, 116831] 00000008:00010000:0.0:1788201706.661906:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.661907:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.661912:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661918:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478539776 478543871) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661923:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661932:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661936:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.661940:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478539776, 478543872) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.661943:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116831 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.661947:0:17463:0:(rw26.c:898:ll_write_end()) pos 478539776, len 4096, copied 4096 00000080:00200000:0.0:1788201706.661949:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000298ac436[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116831] 00000080:00200000:0.0:1788201706.661950:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000aee8f03b fffffc0001029 2:0 ffff93c0c13808c0 116831 lru 00000080:00200000:0.0:1788201706.661954:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008cd67f66 116831: 1< 2 - - > 2< 478539776 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68416 - -> 00000080:00200000:0.0:1788201706.661955:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000298ac436 00000080:00200000:0.0:1788201706.661956:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.661957:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.661963:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.661970:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.661972:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.661973:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.661979:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.661981:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.661982:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.661985:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.661993:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.661998:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.661999:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.662001:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.662003:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116831, 116831] 00000080:00200000:0.0:1788201706.662010:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.662012:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.662049:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.662052:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.662062:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478543872, count: 4096 00000080:00200000:0.0:1788201706.662069:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478543872, bytes: 4096 00000080:00200000:0.0:1788201706.662070:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.662073:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.662074:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478543872, 478547968) 0 0 00000080:00200000:0.0:1788201706.662075:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.662078:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116832, 116832] 00020000:00200000:0.0:1788201706.662080:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478543872 chunk: [478543872, 478547968) 478547968, 4096 00020000:00200000:0.0:1788201706.662082:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.662089:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116832, 116832] 00000008:00010000:0.0:1788201706.662092:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.662093:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.662099:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662105:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478543872 478547967) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662114:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662118:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662122:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.662127:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478543872, 478547968) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.662130:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116832 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.662134:0:17463:0:(rw26.c:898:ll_write_end()) pos 478543872, len 4096, copied 4096 00000080:00200000:0.0:1788201706.662136:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000073214341[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116832] 00000080:00200000:0.0:1788201706.662138:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000076c00be4 fffffc0001029 2:0 ffff93c0c1380ee0 116832 lru 00000080:00200000:0.0:1788201706.662142:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000044d40dd5 116832: 1< 2 - - > 2< 478543872 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68417 - -> 00000080:00200000:0.0:1788201706.662143:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000073214341 00000080:00200000:0.0:1788201706.662144:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.662145:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.662151:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.662158:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.662160:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.662161:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.662168:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.662169:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.662171:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.662174:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662178:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.662183:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.662185:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.662187:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.662189:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116832, 116832] 00000080:00200000:0.0:1788201706.662196:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.662199:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.662227:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.662230:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.662246:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478547968, count: 4096 00000080:00200000:0.0:1788201706.662256:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478547968, bytes: 4096 00000080:00200000:0.0:1788201706.662258:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.662261:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.662263:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478547968, 478552064) 0 0 00000080:00200000:0.0:1788201706.662265:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.662270:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116833, 116833] 00020000:00200000:0.0:1788201706.662273:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478547968 chunk: [478547968, 478552064) 478552064, 4096 00020000:00200000:0.0:1788201706.662275:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.662285:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116833, 116833] 00000008:00010000:0.0:1788201706.662290:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.662292:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.662299:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662312:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478547968 478552063) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662320:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662327:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662333:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.662339:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478547968, 478552064) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.662344:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116833 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.662349:0:17463:0:(rw26.c:898:ll_write_end()) pos 478547968, len 4096, copied 4096 00000080:00200000:0.0:1788201706.662352:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000dbb45b3d[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116833] 00000080:00200000:0.0:1788201706.662355:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000a0b86cc fffffc0001029 2:0 ffff93c0c1381460 116833 lru 00000080:00200000:0.0:1788201706.662360:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007bb928fd 116833: 1< 2 - - > 2< 478547968 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68418 - -> 00000080:00200000:0.0:1788201706.662362:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000dbb45b3d 00000080:00200000:0.0:1788201706.662363:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.662365:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.662372:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.662383:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.662386:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.662388:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.662397:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.662400:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.662402:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.662406:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662413:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.662421:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.662423:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.662428:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.662430:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116833, 116833] 00000080:00200000:0.0:1788201706.662440:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.662443:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.662469:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.662472:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.662487:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478552064, count: 4096 00000080:00200000:0.0:1788201706.662498:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478552064, bytes: 4096 00000080:00200000:0.0:1788201706.662500:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.662504:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.662506:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478552064, 478556160) 0 0 00000080:00200000:0.0:1788201706.662508:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.662513:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116834, 116834] 00020000:00200000:0.0:1788201706.662516:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478552064 chunk: [478552064, 478556160) 478556160, 4096 00020000:00200000:0.0:1788201706.662518:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.662528:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116834, 116834] 00000008:00010000:0.0:1788201706.662532:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.662534:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.662542:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662551:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478552064 478556159) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662560:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662568:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662575:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.662583:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478552064, 478556160) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.662587:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116834 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.662590:0:17463:0:(rw26.c:898:ll_write_end()) pos 478552064, len 4096, copied 4096 00000080:00200000:0.0:1788201706.662593:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000337fa99f[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116834] 00000080:00200000:0.0:1788201706.662595:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c44beef0 fffffc0001029 2:0 ffff93c0c13810e0 116834 lru 00000080:00200000:0.0:1788201706.662598:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001121b8ba 116834: 1< 2 - - > 2< 478552064 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68419 - -> 00000080:00200000:0.0:1788201706.662600:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000337fa99f 00000080:00200000:0.0:1788201706.662601:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.662602:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.662608:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.662615:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.662617:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.662619:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.662625:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.662627:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.662628:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.662631:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662636:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.662641:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.662642:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.662645:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.662647:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116834, 116834] 00000080:00200000:0.0:1788201706.662654:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.662656:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.662673:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.662676:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.662686:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478556160, count: 4096 00000080:00200000:0.0:1788201706.662692:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478556160, bytes: 4096 00000080:00200000:0.0:1788201706.662694:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.662696:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.662697:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478556160, 478560256) 0 0 00000080:00200000:0.0:1788201706.662698:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.662702:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116835, 116835] 00020000:00200000:0.0:1788201706.662703:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478556160 chunk: [478556160, 478560256) 478560256, 4096 00020000:00200000:0.0:1788201706.662705:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.662711:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116835, 116835] 00000008:00010000:0.0:1788201706.662714:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.662716:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.662721:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662727:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478556160 478560255) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662732:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662736:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662740:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.662744:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478556160, 478560256) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.662747:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116835 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.662751:0:17463:0:(rw26.c:898:ll_write_end()) pos 478556160, len 4096, copied 4096 00000080:00200000:0.0:1788201706.662753:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ce3f7d97[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116835] 00000080:00200000:0.0:1788201706.662755:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000739343df fffffc0001029 2:0 ffff93c0c1381700 116835 lru 00000080:00200000:0.0:1788201706.662758:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c93e4613 116835: 1< 2 - - > 2< 478556160 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68420 - -> 00000080:00200000:0.0:1788201706.662760:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ce3f7d97 00000080:00200000:0.0:1788201706.662761:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.662762:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.662767:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.662778:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.662780:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.662781:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.662787:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.662788:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.662790:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.662792:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662797:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.662802:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.662803:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.662806:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.662808:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116835, 116835] 00000080:00200000:0.0:1788201706.662814:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.662816:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.662851:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.662854:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.662867:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478560256, count: 4096 00000080:00200000:0.0:1788201706.662873:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478560256, bytes: 4096 00000080:00200000:0.0:1788201706.662875:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.662877:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.662879:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478560256, 478564352) 0 0 00000080:00200000:0.0:1788201706.662880:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.662883:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116836, 116836] 00020000:00200000:0.0:1788201706.662885:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478560256 chunk: [478560256, 478564352) 478564352, 4096 00020000:00200000:0.0:1788201706.662887:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.662893:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116836, 116836] 00000008:00010000:0.0:1788201706.662896:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.662898:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.662903:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662908:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478560256 478564351) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662914:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662918:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662922:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.662927:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478560256, 478564352) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.662930:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116836 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.662934:0:17463:0:(rw26.c:898:ll_write_end()) pos 478560256, len 4096, copied 4096 00000080:00200000:0.0:1788201706.662936:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000179616b1[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116836] 00000080:00200000:0.0:1788201706.662938:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000158f1b0a fffffc0001029 2:0 ffff93c0c1381000 116836 lru 00000080:00200000:0.0:1788201706.662941:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003ac94251 116836: 1< 2 - - > 2< 478560256 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68421 - -> 00000080:00200000:0.0:1788201706.662943:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000179616b1 00000080:00200000:0.0:1788201706.662944:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.662946:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.662951:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.662959:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.662961:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.662962:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.662968:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.662970:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.662971:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.662974:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.662978:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.662983:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.662984:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.662987:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.662988:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116836, 116836] 00000080:00200000:0.0:1788201706.662995:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.662997:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.663029:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.663032:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.663043:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478564352, count: 4096 00000080:00200000:0.0:1788201706.663050:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478564352, bytes: 4096 00000080:00200000:0.0:1788201706.663052:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.663054:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.663055:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478564352, 478568448) 0 0 00000080:00200000:0.0:1788201706.663057:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.663061:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116837, 116837] 00020000:00200000:0.0:1788201706.663062:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478564352 chunk: [478564352, 478568448) 478568448, 4096 00020000:00200000:0.0:1788201706.663064:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.663071:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116837, 116837] 00000008:00010000:0.0:1788201706.663074:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.663075:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.663081:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663086:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478564352 478568447) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663092:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663096:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663100:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.663105:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478564352, 478568448) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.663108:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116837 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.663112:0:17463:0:(rw26.c:898:ll_write_end()) pos 478564352, len 4096, copied 4096 00000080:00200000:0.0:1788201706.663114:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008f4516b9[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116837] 00000080:00200000:0.0:1788201706.663116:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003ef69292 fffffc0001029 2:0 ffff93c0c1381a80 116837 lru 00000080:00200000:0.0:1788201706.663120:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000016929911 116837: 1< 2 - - > 2< 478564352 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68422 - -> 00000080:00200000:0.0:1788201706.663122:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008f4516b9 00000080:00200000:0.0:1788201706.663123:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.663124:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.663130:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.663137:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.663139:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.663140:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.663146:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.663148:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.663149:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.663152:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663156:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.663161:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.663163:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.663166:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.663168:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116837, 116837] 00000080:00200000:0.0:1788201706.663174:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.663176:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.663194:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.663197:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.663211:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478568448, count: 4096 00000080:00200000:0.0:1788201706.663223:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478568448, bytes: 4096 00000080:00200000:0.0:1788201706.663226:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.663229:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.663231:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478568448, 478572544) 0 0 00000080:00200000:0.0:1788201706.663233:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.663238:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116838, 116838] 00020000:00200000:0.0:1788201706.663241:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478568448 chunk: [478568448, 478572544) 478572544, 4096 00020000:00200000:0.0:1788201706.663243:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.663254:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116838, 116838] 00000008:00010000:0.0:1788201706.663259:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.663261:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.663268:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663277:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478568448 478572543) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663285:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663291:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663296:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.663300:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478568448, 478572544) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.663303:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116838 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.663307:0:17463:0:(rw26.c:898:ll_write_end()) pos 478568448, len 4096, copied 4096 00000080:00200000:0.0:1788201706.663309:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ee13c626[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116838] 00000080:00200000:0.0:1788201706.663311:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000031ccfc5e fffffc0001029 2:0 ffff93c0c1381380 116838 lru 00000080:00200000:0.0:1788201706.663315:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005bcc6d8c 116838: 1< 2 - - > 2< 478568448 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68423 - -> 00000080:00200000:0.0:1788201706.663316:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ee13c626 00000080:00200000:0.0:1788201706.663317:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.663318:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.663324:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.663332:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.663333:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.663335:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.663341:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.663343:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.663344:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.663347:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663352:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.663357:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.663358:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.663361:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.663363:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116838, 116838] 00000080:00200000:0.0:1788201706.663370:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.663371:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.663390:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.663392:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.663402:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478572544, count: 4096 00000080:00200000:0.0:1788201706.663408:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478572544, bytes: 4096 00000080:00200000:0.0:1788201706.663410:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.663412:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.663413:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478572544, 478576640) 0 0 00000080:00200000:0.0:1788201706.663414:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.663417:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116839, 116839] 00020000:00200000:0.0:1788201706.663419:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478572544 chunk: [478572544, 478576640) 478576640, 4096 00020000:00200000:0.0:1788201706.663421:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.663427:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116839, 116839] 00000008:00010000:0.0:1788201706.663430:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.663432:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.663437:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663442:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478572544 478576639) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663447:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663452:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663456:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.663461:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478572544, 478576640) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.663464:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116839 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.663468:0:17463:0:(rw26.c:898:ll_write_end()) pos 478572544, len 4096, copied 4096 00000080:00200000:0.0:1788201706.663471:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002795ba47[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116839] 00000080:00200000:0.0:1788201706.663474:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000093c434d8 fffffc0001029 2:0 ffff93c0c13818c0 116839 lru 00000080:00200000:0.0:1788201706.663479:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f2f207cf 116839: 1< 2 - - > 2< 478572544 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68424 - -> 00000080:00200000:0.0:1788201706.663482:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002795ba47 00000080:00200000:0.0:1788201706.663483:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.663485:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.663493:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.663504:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.663507:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.663509:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.663517:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.663519:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.663521:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.663525:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663532:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.663541:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.663543:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.663547:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.663550:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116839, 116839] 00000080:00200000:0.0:1788201706.663560:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.663563:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.663591:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.663594:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.663607:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478576640, count: 4096 00000080:00200000:0.0:1788201706.663617:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478576640, bytes: 4096 00000080:00200000:0.0:1788201706.663620:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.663624:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.663625:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478576640, 478580736) 0 0 00000080:00200000:0.0:1788201706.663627:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.663632:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116840, 116840] 00020000:00200000:0.0:1788201706.663634:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478576640 chunk: [478576640, 478580736) 478580736, 4096 00020000:00200000:0.0:1788201706.663636:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.663647:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116840, 116840] 00000008:00010000:0.0:1788201706.663652:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.663654:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.663662:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663671:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478576640 478580735) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663677:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663682:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663686:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.663690:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478576640, 478580736) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.663694:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116840 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.663698:0:17463:0:(rw26.c:898:ll_write_end()) pos 478576640, len 4096, copied 4096 00000080:00200000:0.0:1788201706.663701:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005092e9f7[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116840] 00000080:00200000:0.0:1788201706.663702:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005a6eb932 fffffc0001029 2:0 ffff93c0c1381ee0 116840 lru 00000080:00200000:0.0:1788201706.663706:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000043917f1b 116840: 1< 2 - - > 2< 478576640 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68425 - -> 00000080:00200000:0.0:1788201706.663708:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005092e9f7 00000080:00200000:0.0:1788201706.663708:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.663710:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.663715:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.663723:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.663725:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.663726:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.663732:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.663734:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.663736:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.663738:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663743:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.663748:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.663749:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.663751:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.663753:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116840, 116840] 00000080:00200000:0.0:1788201706.663760:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.663762:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.663780:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.663783:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.663793:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478580736, count: 4096 00000080:00200000:0.0:1788201706.663799:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478580736, bytes: 4096 00000080:00200000:0.0:1788201706.663801:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.663803:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.663804:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478580736, 478584832) 0 0 00000080:00200000:0.0:1788201706.663806:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.663809:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116841, 116841] 00020000:00200000:0.0:1788201706.663811:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478580736 chunk: [478580736, 478584832) 478584832, 4096 00020000:00200000:0.0:1788201706.663812:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.663819:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116841, 116841] 00000008:00010000:0.0:1788201706.663822:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.663823:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.663841:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663848:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478580736 478584831) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663854:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663858:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663862:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.663870:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478580736, 478584832) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.663873:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116841 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.663877:0:17463:0:(rw26.c:898:ll_write_end()) pos 478580736, len 4096, copied 4096 00000080:00200000:0.0:1788201706.663879:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009790ca84[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116841] 00000080:00200000:0.0:1788201706.663881:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002d8185eb fffffc0001029 2:0 ffff93c0c13817e0 116841 lru 00000080:00200000:0.0:1788201706.663885:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000830e2e57 116841: 1< 2 - - > 2< 478580736 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68426 - -> 00000080:00200000:0.0:1788201706.663886:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009790ca84 00000080:00200000:0.0:1788201706.663887:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.663888:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.663894:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.663901:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.663903:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.663904:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.663910:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.663912:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.663913:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.663916:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.663921:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.663926:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.663927:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.663930:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.663932:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116841, 116841] 00000080:00200000:0.0:1788201706.663938:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.663940:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.663958:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.663960:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.663970:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478584832, count: 4096 00000080:00200000:0.0:1788201706.663976:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478584832, bytes: 4096 00000080:00200000:0.0:1788201706.663978:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.663980:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.663981:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478584832, 478588928) 0 0 00000080:00200000:0.0:1788201706.663982:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.663986:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116842, 116842] 00020000:00200000:0.0:1788201706.663988:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478584832 chunk: [478584832, 478588928) 478588928, 4096 00020000:00200000:0.0:1788201706.663989:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.663995:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116842, 116842] 00000008:00010000:0.0:1788201706.663998:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.664000:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.664005:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664010:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478584832 478588927) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664029:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664040:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664051:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.664056:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478584832, 478588928) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.664059:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116842 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.664063:0:17463:0:(rw26.c:898:ll_write_end()) pos 478584832, len 4096, copied 4096 00000080:00200000:0.0:1788201706.664066:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004fe61a2d[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116842] 00000080:00200000:0.0:1788201706.664067:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d3f3f043 fffffc0001029 2:0 ffff93c0c13812a0 116842 lru 00000080:00200000:0.0:1788201706.664071:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b12a79cc 116842: 1< 2 - - > 2< 478584832 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68427 - -> 00000080:00200000:0.0:1788201706.664072:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004fe61a2d 00000080:00200000:0.0:1788201706.664073:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.664074:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.664080:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.664088:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.664090:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.664091:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.664097:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.664099:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.664100:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.664103:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664108:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.664116:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.664118:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.664120:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.664122:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116842, 116842] 00000080:00200000:0.0:1788201706.664129:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.664131:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.664149:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.664151:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.664161:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478588928, count: 4096 00000080:00200000:0.0:1788201706.664167:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478588928, bytes: 4096 00000080:00200000:0.0:1788201706.664169:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.664171:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.664172:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478588928, 478593024) 0 0 00000080:00200000:0.0:1788201706.664174:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.664177:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116843, 116843] 00020000:00200000:0.0:1788201706.664179:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478588928 chunk: [478588928, 478593024) 478593024, 4096 00020000:00200000:0.0:1788201706.664180:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.664187:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116843, 116843] 00000008:00010000:0.0:1788201706.664190:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.664191:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.664196:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664204:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478588928 478593023) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664213:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664226:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664233:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.664239:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478588928, 478593024) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.664245:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116843 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.664249:0:17463:0:(rw26.c:898:ll_write_end()) pos 478588928, len 4096, copied 4096 00000080:00200000:0.0:1788201706.664253:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000edcc3ee2[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116843] 00000080:00200000:0.0:1788201706.664256:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000087f81072 fffffc0001029 2:0 ffff93c0c1381e00 116843 lru 00000080:00200000:0.0:1788201706.664261:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000090498d72 116843: 1< 2 - - > 2< 478588928 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68428 - -> 00000080:00200000:0.0:1788201706.664263:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000edcc3ee2 00000080:00200000:0.0:1788201706.664265:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.664266:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.664274:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.664284:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.664287:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.664289:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.664297:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.664300:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.664302:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.664305:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664317:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.664324:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.664326:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.664329:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.664332:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116843, 116843] 00000080:00200000:0.0:1788201706.664341:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.664344:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.664371:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.664374:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.664390:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478593024, count: 4096 00000080:00200000:0.0:1788201706.664401:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478593024, bytes: 4096 00000080:00200000:0.0:1788201706.664404:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.664407:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.664408:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478593024, 478597120) 0 0 00000080:00200000:0.0:1788201706.664411:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.664415:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116844, 116844] 00020000:00200000:0.0:1788201706.664418:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478593024 chunk: [478593024, 478597120) 478597120, 4096 00020000:00200000:0.0:1788201706.664420:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.664431:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116844, 116844] 00000008:00010000:0.0:1788201706.664436:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.664438:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.664446:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664455:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478593024 478597119) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664467:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664471:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664475:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.664480:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478593024, 478597120) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.664483:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116844 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.664487:0:17463:0:(rw26.c:898:ll_write_end()) pos 478593024, len 4096, copied 4096 00000080:00200000:0.0:1788201706.664489:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006d19d630[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116844] 00000080:00200000:0.0:1788201706.664491:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fc41c3ad fffffc0001029 2:0 ffff93c0c13811c0 116844 lru 00000080:00200000:0.0:1788201706.664494:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005aee0ce0 116844: 1< 2 - - > 2< 478593024 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68429 - -> 00000080:00200000:0.0:1788201706.664495:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006d19d630 00000080:00200000:0.0:1788201706.664496:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.664498:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.664503:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.664511:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.664513:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.664514:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.664520:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.664522:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.664524:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.664527:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664532:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.664537:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.664538:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.664541:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.664543:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116844, 116844] 00000080:00200000:0.0:1788201706.664550:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.664552:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.664570:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.664572:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.664582:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478597120, count: 4096 00000080:00200000:0.0:1788201706.664588:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478597120, bytes: 4096 00000080:00200000:0.0:1788201706.664590:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.664592:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.664593:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478597120, 478601216) 0 0 00000080:00200000:0.0:1788201706.664595:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.664598:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116845, 116845] 00020000:00200000:0.0:1788201706.664600:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478597120 chunk: [478597120, 478601216) 478601216, 4096 00020000:00200000:0.0:1788201706.664601:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.664607:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116845, 116845] 00000008:00010000:0.0:1788201706.664610:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.664612:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.664617:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664626:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478597120 478601215) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664631:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664635:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664639:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.664644:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478597120, 478601216) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.664647:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116845 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.664651:0:17463:0:(rw26.c:898:ll_write_end()) pos 478597120, len 4096, copied 4096 00000080:00200000:0.0:1788201706.664653:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e5af598d[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116845] 00000080:00200000:0.0:1788201706.664655:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000002f09a9 fffffc0001029 2:0 ffff93c0c1381620 116845 lru 00000080:00200000:0.0:1788201706.664659:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000074e5595 116845: 1< 2 - - > 2< 478597120 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68430 - -> 00000080:00200000:0.0:1788201706.664660:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e5af598d 00000080:00200000:0.0:1788201706.664661:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.664662:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.664668:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.664675:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.664677:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.664678:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.664684:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.664686:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.664688:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.664690:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664694:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.664699:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.664700:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.664703:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.664705:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116845, 116845] 00000080:00200000:0.0:1788201706.664711:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.664713:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.664731:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.664733:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.664743:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478601216, count: 4096 00000080:00200000:0.0:1788201706.664749:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478601216, bytes: 4096 00000080:00200000:0.0:1788201706.664750:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.664753:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.664754:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478601216, 478605312) 0 0 00000080:00200000:0.0:1788201706.664755:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.664758:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116846, 116846] 00020000:00200000:0.0:1788201706.664760:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478601216 chunk: [478601216, 478605312) 478605312, 4096 00020000:00200000:0.0:1788201706.664762:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.664768:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116846, 116846] 00000008:00010000:0.0:1788201706.664771:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.664772:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.664777:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664783:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478601216 478605311) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664789:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664793:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664797:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.664801:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478601216, 478605312) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.664804:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116846 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.664808:0:17463:0:(rw26.c:898:ll_write_end()) pos 478601216, len 4096, copied 4096 00000080:00200000:0.0:1788201706.664810:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f14d1510[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116846] 00000080:00200000:0.0:1788201706.664812:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006334de53 fffffc0001029 2:0 ffff93c0c1381d20 116846 lru 00000080:00200000:0.0:1788201706.664816:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f5648584 116846: 1< 2 - - > 2< 478601216 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68431 - -> 00000080:00200000:0.0:1788201706.664817:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f14d1510 00000080:00200000:0.0:1788201706.664818:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.664819:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.664825:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.664846:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.664848:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.664849:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.664855:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.664857:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.664859:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.664861:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664866:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.664871:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.664873:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.664877:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.664879:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116846, 116846] 00000080:00200000:0.0:1788201706.664889:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.664892:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.664921:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.664925:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.664939:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478605312, count: 4096 00000080:00200000:0.0:1788201706.664948:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478605312, bytes: 4096 00000080:00200000:0.0:1788201706.664950:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.664953:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.664955:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478605312, 478609408) 0 0 00000080:00200000:0.0:1788201706.664957:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.664961:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116847, 116847] 00020000:00200000:0.0:1788201706.664964:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478605312 chunk: [478605312, 478609408) 478609408, 4096 00020000:00200000:0.0:1788201706.664966:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.664976:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116847, 116847] 00000008:00010000:0.0:1788201706.664981:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.664984:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.664990:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.664999:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478605312 478609407) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665007:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665028:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665033:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.665039:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478605312, 478609408) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.665043:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116847 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.665046:0:17463:0:(rw26.c:898:ll_write_end()) pos 478605312, len 4096, copied 4096 00000080:00200000:0.0:1788201706.665049:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000027c2f5d0[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116847] 00000080:00200000:0.0:1788201706.665050:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006d7dd6bc fffffc0001029 2:0 ffff93c0c13819a0 116847 lru 00000080:00200000:0.0:1788201706.665054:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000093878b35 116847: 1< 2 - - > 2< 478605312 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68432 - -> 00000080:00200000:0.0:1788201706.665055:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000027c2f5d0 00000080:00200000:0.0:1788201706.665056:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.665057:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.665064:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.665075:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.665077:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.665078:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.665085:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.665086:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.665088:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.665091:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665095:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.665101:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.665102:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.665105:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.665107:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116847, 116847] 00000080:00200000:0.0:1788201706.665114:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.665116:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.665134:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.665137:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.665147:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478609408, count: 4096 00000080:00200000:0.0:1788201706.665153:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478609408, bytes: 4096 00000080:00200000:0.0:1788201706.665155:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.665157:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.665158:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478609408, 478613504) 0 0 00000080:00200000:0.0:1788201706.665160:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.665163:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116848, 116848] 00020000:00200000:0.0:1788201706.665165:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478609408 chunk: [478609408, 478613504) 478613504, 4096 00020000:00200000:0.0:1788201706.665167:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.665173:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116848, 116848] 00000008:00010000:0.0:1788201706.665176:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.665178:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.665183:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665188:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478609408 478613503) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665194:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665200:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665207:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.665215:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478609408, 478613504) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.665219:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116848 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.665225:0:17463:0:(rw26.c:898:ll_write_end()) pos 478609408, len 4096, copied 4096 00000080:00200000:0.0:1788201706.665229:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000028c82cb8[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116848] 00000080:00200000:0.0:1788201706.665232:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004104d028 fffffc0001029 2:0 ffff93c0c1381b60 116848 lru 00000080:00200000:0.0:1788201706.665237:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000065456057 116848: 1< 2 - - > 2< 478609408 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68433 - -> 00000080:00200000:0.0:1788201706.665239:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000028c82cb8 00000080:00200000:0.0:1788201706.665241:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.665243:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.665252:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.665263:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.665266:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.665267:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.665274:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.665276:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.665278:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.665280:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665285:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.665290:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.665291:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.665294:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.665296:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116848, 116848] 00000080:00200000:0.0:1788201706.665303:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.665305:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.665323:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.665326:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.665336:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478613504, count: 4096 00000080:00200000:0.0:1788201706.665342:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478613504, bytes: 4096 00000080:00200000:0.0:1788201706.665344:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.665346:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.665347:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478613504, 478617600) 0 0 00000080:00200000:0.0:1788201706.665349:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.665352:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116849, 116849] 00020000:00200000:0.0:1788201706.665354:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478613504 chunk: [478613504, 478617600) 478617600, 4096 00020000:00200000:0.0:1788201706.665356:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.665367:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116849, 116849] 00000008:00010000:0.0:1788201706.665372:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.665374:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.665381:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665390:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478613504 478617599) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665399:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665407:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665414:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.665420:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478613504, 478617600) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.665423:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116849 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.665427:0:17463:0:(rw26.c:898:ll_write_end()) pos 478613504, len 4096, copied 4096 00000080:00200000:0.0:1788201706.665430:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000004231640[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116849] 00000080:00200000:0.0:1788201706.665432:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003a79a685 fffffc0001029 2:0 ffff93c0c1381c40 116849 lru 00000080:00200000:0.0:1788201706.665435:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a582492d 116849: 1< 2 - - > 2< 478613504 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68434 - -> 00000080:00200000:0.0:1788201706.665438:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000004231640 00000080:00200000:0.0:1788201706.665439:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.665440:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.665446:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.665454:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.665456:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.665457:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.665463:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.665465:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.665466:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.665469:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665474:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.665479:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.665480:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.665482:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.665484:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116849, 116849] 00000080:00200000:0.0:1788201706.665492:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.665494:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.665512:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.665514:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.665525:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478617600, count: 4096 00000080:00200000:0.0:1788201706.665530:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478617600, bytes: 4096 00000080:00200000:0.0:1788201706.665533:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.665535:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.665536:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478617600, 478621696) 0 0 00000080:00200000:0.0:1788201706.665537:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.665541:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116850, 116850] 00020000:00200000:0.0:1788201706.665542:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478617600 chunk: [478617600, 478621696) 478621696, 4096 00020000:00200000:0.0:1788201706.665544:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.665550:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116850, 116850] 00000008:00010000:0.0:1788201706.665553:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.665555:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.665560:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665565:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478617600 478621695) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665571:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665575:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665579:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.665583:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478617600, 478621696) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.665586:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116850 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.665590:0:17463:0:(rw26.c:898:ll_write_end()) pos 478617600, len 4096, copied 4096 00000080:00200000:0.0:1788201706.665592:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000047548125[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116850] 00000080:00200000:0.0:1788201706.665594:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000016c2e26b fffffc0001029 2:0 ffff93c0c1381540 116850 lru 00000080:00200000:0.0:1788201706.665598:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004d8f4600 116850: 1< 2 - - > 2< 478617600 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68435 - -> 00000080:00200000:0.0:1788201706.665599:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000047548125 00000080:00200000:0.0:1788201706.665600:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.665601:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.665606:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.665613:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.665615:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.665616:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.665622:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.665624:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.665625:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.665627:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665632:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.665637:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.665638:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.665640:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.665642:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116850, 116850] 00000080:00200000:0.0:1788201706.665649:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.665651:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.665668:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.665670:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.665680:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478621696, count: 4096 00000080:00200000:0.0:1788201706.665686:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478621696, bytes: 4096 00000080:00200000:0.0:1788201706.665688:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.665690:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.665691:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478621696, 478625792) 0 0 00000080:00200000:0.0:1788201706.665692:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.665696:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116851, 116851] 00020000:00200000:0.0:1788201706.665697:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478621696 chunk: [478621696, 478625792) 478625792, 4096 00020000:00200000:0.0:1788201706.665699:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.665705:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116851, 116851] 00000008:00010000:0.0:1788201706.665708:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.665709:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.665714:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665720:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478621696 478625791) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665725:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665730:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665734:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.665738:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478621696, 478625792) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.665741:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116851 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.665745:0:17463:0:(rw26.c:898:ll_write_end()) pos 478621696, len 4096, copied 4096 00000080:00200000:0.0:1788201706.665748:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000409d366f[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116851] 00000080:00200000:0.0:1788201706.665749:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e38d7619 fffffc0001029 2:0 ffff93c0c13822a0 116851 lru 00000080:00200000:0.0:1788201706.665753:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008f847ea2 116851: 1< 2 - - > 2< 478621696 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68436 - -> 00000080:00200000:0.0:1788201706.665754:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000409d366f 00000080:00200000:0.0:1788201706.665755:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.665756:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.665761:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.665769:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.665770:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.665772:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.665777:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.665779:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.665780:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.665783:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665787:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.665792:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.665793:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.665796:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.665798:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116851, 116851] 00000080:00200000:0.0:1788201706.665804:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.665806:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.665823:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.665826:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.665850:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478625792, count: 4096 00000080:00200000:0.0:1788201706.665856:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478625792, bytes: 4096 00000080:00200000:0.0:1788201706.665858:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.665860:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.665861:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478625792, 478629888) 0 0 00000080:00200000:0.0:1788201706.665863:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.665866:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116852, 116852] 00020000:00200000:0.0:1788201706.665868:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478625792 chunk: [478625792, 478629888) 478629888, 4096 00020000:00200000:0.0:1788201706.665869:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.665876:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116852, 116852] 00000008:00010000:0.0:1788201706.665879:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.665880:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.665885:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665891:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478625792 478629887) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665896:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665901:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665905:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.665909:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478625792, 478629888) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.665913:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116852 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.665916:0:17463:0:(rw26.c:898:ll_write_end()) pos 478625792, len 4096, copied 4096 00000080:00200000:0.0:1788201706.665919:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f19b05ce[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116852] 00000080:00200000:0.0:1788201706.665920:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001da9eb12 fffffc0001029 2:0 ffff93c0c1382e00 116852 lru 00000080:00200000:0.0:1788201706.665924:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c242532a 116852: 1< 2 - - > 2< 478625792 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68437 - -> 00000080:00200000:0.0:1788201706.665925:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f19b05ce 00000080:00200000:0.0:1788201706.665926:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.665928:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.665933:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.665941:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.665942:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.665943:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.665949:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.665951:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.665953:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.665955:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.665960:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.665965:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.665966:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.665968:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.665970:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116852, 116852] 00000080:00200000:0.0:1788201706.665977:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.665978:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.665996:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.665998:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.666008:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478629888, count: 4096 00000080:00200000:0.0:1788201706.666028:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478629888, bytes: 4096 00000080:00200000:0.0:1788201706.666032:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.666034:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.666035:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478629888, 478633984) 0 0 00000080:00200000:0.0:1788201706.666037:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.666040:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116853, 116853] 00020000:00200000:0.0:1788201706.666042:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478629888 chunk: [478629888, 478633984) 478633984, 4096 00020000:00200000:0.0:1788201706.666044:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.666052:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116853, 116853] 00000008:00010000:0.0:1788201706.666055:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.666056:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.666062:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666068:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478629888 478633983) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666073:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666077:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666081:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.666089:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478629888, 478633984) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.666093:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116853 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.666096:0:17463:0:(rw26.c:898:ll_write_end()) pos 478629888, len 4096, copied 4096 00000080:00200000:0.0:1788201706.666098:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000dd546e04[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116853] 00000080:00200000:0.0:1788201706.666101:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ce2bc9bf fffffc0001029 2:0 ffff93c0c13821c0 116853 lru 00000080:00200000:0.0:1788201706.666105:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000fc163a2 116853: 1< 2 - - > 2< 478629888 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68438 - -> 00000080:00200000:0.0:1788201706.666106:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000dd546e04 00000080:00200000:0.0:1788201706.666107:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.666108:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.666113:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.666121:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.666123:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.666124:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.666130:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.666132:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.666133:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.666136:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666141:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.666146:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.666147:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.666150:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.666152:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116853, 116853] 00000080:00200000:0.0:1788201706.666159:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.666161:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.666178:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.666181:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.666191:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478633984, count: 4096 00000080:00200000:0.0:1788201706.666198:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478633984, bytes: 4096 00000080:00200000:0.0:1788201706.666200:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.666203:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.666205:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478633984, 478638080) 0 0 00000080:00200000:0.0:1788201706.666207:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.666212:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116854, 116854] 00020000:00200000:0.0:1788201706.666214:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478633984 chunk: [478633984, 478638080) 478638080, 4096 00020000:00200000:0.0:1788201706.666216:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.666228:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116854, 116854] 00000008:00010000:0.0:1788201706.666233:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.666236:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.666243:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666251:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478633984 478638079) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666257:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666261:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666269:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.666274:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478633984, 478638080) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.666277:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116854 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.666281:0:17463:0:(rw26.c:898:ll_write_end()) pos 478633984, len 4096, copied 4096 00000080:00200000:0.0:1788201706.666284:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000018b6863b[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116854] 00000080:00200000:0.0:1788201706.666285:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d6015502 fffffc0001029 2:0 ffff93c0c1382620 116854 lru 00000080:00200000:0.0:1788201706.666289:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000789e3658 116854: 1< 2 - - > 2< 478633984 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68439 - -> 00000080:00200000:0.0:1788201706.666290:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000018b6863b 00000080:00200000:0.0:1788201706.666291:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.666292:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.666298:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.666306:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.666307:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.666309:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.666315:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.666316:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.666318:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.666320:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666325:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.666334:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.666335:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.666338:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.666340:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116854, 116854] 00000080:00200000:0.0:1788201706.666347:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.666349:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.666374:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.666378:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.666393:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478638080, count: 4096 00000080:00200000:0.0:1788201706.666404:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478638080, bytes: 4096 00000080:00200000:0.0:1788201706.666407:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.666410:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.666411:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478638080, 478642176) 0 0 00000080:00200000:0.0:1788201706.666413:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.666417:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116855, 116855] 00020000:00200000:0.0:1788201706.666419:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478638080 chunk: [478638080, 478642176) 478642176, 4096 00020000:00200000:0.0:1788201706.666421:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.666428:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116855, 116855] 00000008:00010000:0.0:1788201706.666431:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.666433:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.666438:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666444:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478638080 478642175) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666450:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666458:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666462:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.666466:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478638080, 478642176) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.666469:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116855 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.666473:0:17463:0:(rw26.c:898:ll_write_end()) pos 478638080, len 4096, copied 4096 00000080:00200000:0.0:1788201706.666475:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b6f68d6f[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116855] 00000080:00200000:0.0:1788201706.666477:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000089849085 fffffc0001029 2:0 ffff93c0c1382d20 116855 lru 00000080:00200000:0.0:1788201706.666480:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e0e90c06 116855: 1< 2 - - > 2< 478638080 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68440 - -> 00000080:00200000:0.0:1788201706.666481:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b6f68d6f 00000080:00200000:0.0:1788201706.666482:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.666483:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.666489:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.666497:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.666498:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.666500:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.666506:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.666507:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.666509:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.666511:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666519:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.666524:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.666526:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.666528:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.666530:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116855, 116855] 00000080:00200000:0.0:1788201706.666537:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.666538:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.666556:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.666558:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.666569:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478642176, count: 4096 00000080:00200000:0.0:1788201706.666574:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478642176, bytes: 4096 00000080:00200000:0.0:1788201706.666576:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.666578:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.666579:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478642176, 478646272) 0 0 00000080:00200000:0.0:1788201706.666581:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.666584:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116856, 116856] 00020000:00200000:0.0:1788201706.666586:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478642176 chunk: [478642176, 478646272) 478646272, 4096 00020000:00200000:0.0:1788201706.666587:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.666593:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116856, 116856] 00000008:00010000:0.0:1788201706.666596:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.666598:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.666603:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666608:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478642176 478646271) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666617:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666621:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666625:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.666630:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478642176, 478646272) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.666633:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116856 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.666636:0:17463:0:(rw26.c:898:ll_write_end()) pos 478642176, len 4096, copied 4096 00000080:00200000:0.0:1788201706.666639:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000eca3c1d6[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116856] 00000080:00200000:0.0:1788201706.666640:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005f517bf2 fffffc0001029 2:0 ffff93c0c13829a0 116856 lru 00000080:00200000:0.0:1788201706.666644:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006eb6b2ad 116856: 1< 2 - - > 2< 478642176 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68441 - -> 00000080:00200000:0.0:1788201706.666645:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000eca3c1d6 00000080:00200000:0.0:1788201706.666646:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.666648:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.666653:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.666660:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.666662:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.666663:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.666669:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.666671:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.666673:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.666675:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666680:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.666684:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.666686:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.666688:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.666690:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116856, 116856] 00000080:00200000:0.0:1788201706.666697:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.666698:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.666716:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.666718:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.666728:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478646272, count: 4096 00000080:00200000:0.0:1788201706.666734:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478646272, bytes: 4096 00000080:00200000:0.0:1788201706.666735:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.666738:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.666739:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478646272, 478650368) 0 0 00000080:00200000:0.0:1788201706.666740:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.666743:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116857, 116857] 00020000:00200000:0.0:1788201706.666745:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478646272 chunk: [478646272, 478650368) 478650368, 4096 00020000:00200000:0.0:1788201706.666746:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.666753:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116857, 116857] 00000008:00010000:0.0:1788201706.666756:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.666757:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.666762:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666771:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478646272 478650367) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666776:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666780:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666784:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.666789:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478646272, 478650368) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.666792:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116857 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.666795:0:17463:0:(rw26.c:898:ll_write_end()) pos 478646272, len 4096, copied 4096 00000080:00200000:0.0:1788201706.666798:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a0b4bd0e[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116857] 00000080:00200000:0.0:1788201706.666800:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003751a3d2 fffffc0001029 2:0 ffff93c0c1382b60 116857 lru 00000080:00200000:0.0:1788201706.666803:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009517da72 116857: 1< 2 - - > 2< 478646272 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68442 - -> 00000080:00200000:0.0:1788201706.666805:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a0b4bd0e 00000080:00200000:0.0:1788201706.666805:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.666807:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.666812:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.666819:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.666821:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.666822:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.666845:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.666847:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.666849:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.666851:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666856:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.666861:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.666863:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.666866:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.666868:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116857, 116857] 00000080:00200000:0.0:1788201706.666878:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.666880:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.666898:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.666901:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.666911:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478650368, count: 4096 00000080:00200000:0.0:1788201706.666917:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478650368, bytes: 4096 00000080:00200000:0.0:1788201706.666918:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.666921:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.666922:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478650368, 478654464) 0 0 00000080:00200000:0.0:1788201706.666923:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.666926:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116858, 116858] 00020000:00200000:0.0:1788201706.666928:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478650368 chunk: [478650368, 478654464) 478654464, 4096 00020000:00200000:0.0:1788201706.666930:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.666936:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116858, 116858] 00000008:00010000:0.0:1788201706.666939:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.666941:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.666946:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666951:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478650368 478654463) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666957:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666961:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.666965:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.666970:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478650368, 478654464) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.666973:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116858 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.666976:0:17463:0:(rw26.c:898:ll_write_end()) pos 478650368, len 4096, copied 4096 00000080:00200000:0.0:1788201706.666979:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000026cc148e[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116858] 00000080:00200000:0.0:1788201706.666980:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005d0534fb fffffc0001029 2:0 ffff93c0c1382c40 116858 lru 00000080:00200000:0.0:1788201706.666984:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000720b331 116858: 1< 2 - - > 2< 478650368 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68443 - -> 00000080:00200000:0.0:1788201706.666985:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000026cc148e 00000080:00200000:0.0:1788201706.666986:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.666987:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.666993:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.667000:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.667002:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.667003:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.667009:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.667010:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.667012:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.667029:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667035:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.667040:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.667042:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.667044:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.667046:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116858, 116858] 00000080:00200000:0.0:1788201706.667054:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.667055:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.667080:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.667083:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.667100:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478654464, count: 4096 00000080:00200000:0.0:1788201706.667111:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478654464, bytes: 4096 00000080:00200000:0.0:1788201706.667114:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.667117:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.667119:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478654464, 478658560) 0 0 00000080:00200000:0.0:1788201706.667121:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.667126:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116859, 116859] 00020000:00200000:0.0:1788201706.667129:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478654464 chunk: [478654464, 478658560) 478658560, 4096 00020000:00200000:0.0:1788201706.667131:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.667143:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116859, 116859] 00000008:00010000:0.0:1788201706.667149:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.667152:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.667159:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667167:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478654464 478658559) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667173:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667178:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667182:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.667187:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478654464, 478658560) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.667190:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116859 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.667194:0:17463:0:(rw26.c:898:ll_write_end()) pos 478654464, len 4096, copied 4096 00000080:00200000:0.0:1788201706.667197:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b2c6e1eb[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116859] 00000080:00200000:0.0:1788201706.667199:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000036cc9f21 fffffc0001029 2:0 ffff93c0c1382540 116859 lru 00000080:00200000:0.0:1788201706.667205:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001ee1f650 116859: 1< 2 - - > 2< 478654464 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68444 - -> 00000080:00200000:0.0:1788201706.667207:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b2c6e1eb 00000080:00200000:0.0:1788201706.667208:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.667210:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.667219:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.667235:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.667239:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.667241:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.667250:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.667253:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.667255:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.667260:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667268:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.667276:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.667279:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.667282:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.667284:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116859, 116859] 00000080:00200000:0.0:1788201706.667292:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.667294:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.667313:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.667315:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.667326:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478658560, count: 4096 00000080:00200000:0.0:1788201706.667332:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478658560, bytes: 4096 00000080:00200000:0.0:1788201706.667334:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.667336:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.667337:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478658560, 478662656) 0 0 00000080:00200000:0.0:1788201706.667339:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.667342:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116860, 116860] 00020000:00200000:0.0:1788201706.667344:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478658560 chunk: [478658560, 478662656) 478662656, 4096 00020000:00200000:0.0:1788201706.667345:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.667352:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116860, 116860] 00000008:00010000:0.0:1788201706.667356:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.667358:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.667366:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667375:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478658560 478662655) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667383:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667391:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667399:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.667408:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478658560, 478662656) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.667412:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116860 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.667416:0:17463:0:(rw26.c:898:ll_write_end()) pos 478658560, len 4096, copied 4096 00000080:00200000:0.0:1788201706.667418:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bffc0bb2[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116860] 00000080:00200000:0.0:1788201706.667420:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001611061f fffffc0001029 2:0 ffff93c0c1382460 116860 lru 00000080:00200000:0.0:1788201706.667424:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000073bee6d5 116860: 1< 2 - - > 2< 478658560 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68445 - -> 00000080:00200000:0.0:1788201706.667425:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bffc0bb2 00000080:00200000:0.0:1788201706.667426:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.667428:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.667434:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.667442:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.667444:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.667445:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.667452:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.667454:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.667455:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.667458:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667463:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.667468:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.667469:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.667471:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.667473:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116860, 116860] 00000080:00200000:0.0:1788201706.667480:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.667482:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.667501:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.667503:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.667513:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478662656, count: 4096 00000080:00200000:0.0:1788201706.667519:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478662656, bytes: 4096 00000080:00200000:0.0:1788201706.667521:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.667523:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.667524:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478662656, 478666752) 0 0 00000080:00200000:0.0:1788201706.667526:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.667529:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116861, 116861] 00020000:00200000:0.0:1788201706.667531:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478662656 chunk: [478662656, 478666752) 478666752, 4096 00020000:00200000:0.0:1788201706.667532:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.667539:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116861, 116861] 00000008:00010000:0.0:1788201706.667542:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.667544:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.667549:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667554:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478662656 478666751) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667559:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667564:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667568:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.667572:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478662656, 478666752) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.667574:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116861 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.667578:0:17463:0:(rw26.c:898:ll_write_end()) pos 478662656, len 4096, copied 4096 00000080:00200000:0.0:1788201706.667580:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000061e99a10[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116861] 00000080:00200000:0.0:1788201706.667582:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000094819a17 fffffc0001029 2:0 ffff93c0c13820e0 116861 lru 00000080:00200000:0.0:1788201706.667585:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bcb29383 116861: 1< 2 - - > 2< 478662656 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68446 - -> 00000080:00200000:0.0:1788201706.667588:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000061e99a10 00000080:00200000:0.0:1788201706.667589:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.667590:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.667595:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.667603:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.667604:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.667606:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.667611:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.667613:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.667615:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.667617:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667622:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.667626:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.667628:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.667630:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.667632:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116861, 116861] 00000080:00200000:0.0:1788201706.667638:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.667640:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.667658:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.667660:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.667670:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478666752, count: 4096 00000080:00200000:0.0:1788201706.667675:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478666752, bytes: 4096 00000080:00200000:0.0:1788201706.667677:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.667679:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.667681:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478666752, 478670848) 0 0 00000080:00200000:0.0:1788201706.667682:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.667685:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116862, 116862] 00020000:00200000:0.0:1788201706.667687:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478666752 chunk: [478666752, 478670848) 478670848, 4096 00020000:00200000:0.0:1788201706.667688:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.667695:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116862, 116862] 00000008:00010000:0.0:1788201706.667698:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.667699:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.667704:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667709:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478666752 478670847) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667715:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667719:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667723:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.667728:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478666752, 478670848) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.667731:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116862 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.667734:0:17463:0:(rw26.c:898:ll_write_end()) pos 478666752, len 4096, copied 4096 00000080:00200000:0.0:1788201706.667736:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005b67b754[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116862] 00000080:00200000:0.0:1788201706.667738:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001576d900 fffffc0001029 2:0 ffff93c0c1382700 116862 lru 00000080:00200000:0.0:1788201706.667742:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001277226b 116862: 1< 2 - - > 2< 478666752 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68447 - -> 00000080:00200000:0.0:1788201706.667743:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005b67b754 00000080:00200000:0.0:1788201706.667744:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.667745:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.667750:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.667757:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.667759:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.667760:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.667766:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.667768:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.667769:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.667771:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667776:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.667780:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.667782:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.667784:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.667786:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116862, 116862] 00000080:00200000:0.0:1788201706.667793:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.667794:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.667812:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.667814:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.667824:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478670848, count: 4096 00000080:00200000:0.0:1788201706.667843:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478670848, bytes: 4096 00000080:00200000:0.0:1788201706.667845:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.667847:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.667848:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478670848, 478674944) 0 0 00000080:00200000:0.0:1788201706.667850:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.667853:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116863, 116863] 00020000:00200000:0.0:1788201706.667855:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478670848 chunk: [478670848, 478674944) 478674944, 4096 00020000:00200000:0.0:1788201706.667857:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.667863:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116863, 116863] 00000008:00010000:0.0:1788201706.667866:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.667868:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.667873:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667878:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478670848 478674943) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667884:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667888:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667892:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.667897:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478670848, 478674944) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.667900:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116863 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.667904:0:17463:0:(rw26.c:898:ll_write_end()) pos 478670848, len 4096, copied 4096 00000080:00200000:0.0:1788201706.667906:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000383526ed[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116863] 00000080:00200000:0.0:1788201706.667908:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008a4ed033 fffffc0001029 2:0 ffff93c0c1382000 116863 lru 00000080:00200000:0.0:1788201706.667912:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004fc0ab09 116863: 1< 2 - - > 2< 478670848 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68448 - -> 00000080:00200000:0.0:1788201706.667913:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000383526ed 00000080:00200000:0.0:1788201706.667914:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.667915:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.667920:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.667928:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.667930:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.667931:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.667937:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.667938:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.667940:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.667942:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.667947:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.667952:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.667954:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.667956:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.667958:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116863, 116863] 00000080:00200000:0.0:1788201706.667964:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.667966:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.667984:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.667986:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.667996:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478674944, count: 4096 00000080:00200000:0.0:1788201706.668002:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478674944, bytes: 4096 00000080:00200000:0.0:1788201706.668003:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.668005:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.668007:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478674944, 478679040) 0 0 00000080:00200000:0.0:1788201706.668008:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.668011:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116864, 116864] 00020000:00200000:0.0:1788201706.668013:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478674944 chunk: [478674944, 478679040) 478679040, 4096 00020000:00200000:0.0:1788201706.668029:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.668038:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116864, 116864] 00000008:00010000:0.0:1788201706.668041:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.668043:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.668048:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668054:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478674944 478679039) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668059:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668064:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668068:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.668073:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478674944, 478679040) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.668076:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116864 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.668080:0:17463:0:(rw26.c:898:ll_write_end()) pos 478674944, len 4096, copied 4096 00000080:00200000:0.0:1788201706.668083:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b5d16708[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116864] 00000080:00200000:0.0:1788201706.668084:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ff7707e0 fffffc0001029 2:0 ffff93c0c1382a80 116864 lru 00000080:00200000:0.0:1788201706.668088:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005f546a1a 116864: 1< 2 - - > 2< 478674944 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68449 - -> 00000080:00200000:0.0:1788201706.668090:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b5d16708 00000080:00200000:0.0:1788201706.668090:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.668091:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.668097:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.668105:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.668106:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.668107:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.668113:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.668115:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.668117:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.668119:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668123:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.668128:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.668129:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.668132:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.668134:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116864, 116864] 00000080:00200000:0.0:1788201706.668141:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.668142:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.668160:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.668163:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.668173:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478679040, count: 4096 00000080:00200000:0.0:1788201706.668179:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478679040, bytes: 4096 00000080:00200000:0.0:1788201706.668180:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.668183:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.668184:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478679040, 478683136) 0 0 00000080:00200000:0.0:1788201706.668185:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.668188:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116865, 116865] 00020000:00200000:0.0:1788201706.668191:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478679040 chunk: [478679040, 478683136) 478683136, 4096 00020000:00200000:0.0:1788201706.668192:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.668200:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116865, 116865] 00000008:00010000:0.0:1788201706.668205:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.668207:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.668215:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668223:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478679040 478683135) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668232:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668239:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668245:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.668257:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478679040, 478683136) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.668262:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116865 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.668267:0:17463:0:(rw26.c:898:ll_write_end()) pos 478679040, len 4096, copied 4096 00000080:00200000:0.0:1788201706.668272:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000140d5873[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116865] 00000080:00200000:0.0:1788201706.668275:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ba9771fb fffffc0001029 2:0 ffff93c0c1382380 116865 lru 00000080:00200000:0.0:1788201706.668279:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005a6ebd88 116865: 1< 2 - - > 2< 478679040 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68450 - -> 00000080:00200000:0.0:1788201706.668280:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000140d5873 00000080:00200000:0.0:1788201706.668281:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.668283:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.668289:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.668297:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.668299:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.668300:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.668306:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.668308:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.668310:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.668312:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668317:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.668322:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.668324:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.668327:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.668329:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116865, 116865] 00000080:00200000:0.0:1788201706.668336:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.668338:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.668363:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.668366:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.668382:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478683136, count: 4096 00000080:00200000:0.0:1788201706.668392:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478683136, bytes: 4096 00000080:00200000:0.0:1788201706.668395:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.668398:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.668399:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478683136, 478687232) 0 0 00000080:00200000:0.0:1788201706.668400:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.668404:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116866, 116866] 00020000:00200000:0.0:1788201706.668406:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478683136 chunk: [478683136, 478687232) 478687232, 4096 00020000:00200000:0.0:1788201706.668407:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.668415:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116866, 116866] 00000008:00010000:0.0:1788201706.668418:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.668420:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.668425:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668431:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478683136 478687231) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668436:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668441:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668449:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.668453:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478683136, 478687232) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.668456:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116866 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.668460:0:17463:0:(rw26.c:898:ll_write_end()) pos 478683136, len 4096, copied 4096 00000080:00200000:0.0:1788201706.668462:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000958cec9a[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116866] 00000080:00200000:0.0:1788201706.668463:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000eab62485 fffffc0001029 2:0 ffff93c0c13828c0 116866 lru 00000080:00200000:0.0:1788201706.668467:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006b540d9f 116866: 1< 2 - - > 2< 478683136 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68451 - -> 00000080:00200000:0.0:1788201706.668468:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000958cec9a 00000080:00200000:0.0:1788201706.668469:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.668470:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.668476:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.668483:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.668485:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.668486:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.668492:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.668494:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.668496:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.668498:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668503:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.668511:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.668513:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.668515:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.668517:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116866, 116866] 00000080:00200000:0.0:1788201706.668524:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.668526:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.668544:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.668547:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.668557:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478687232, count: 4096 00000080:00200000:0.0:1788201706.668562:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478687232, bytes: 4096 00000080:00200000:0.0:1788201706.668564:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.668566:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.668567:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478687232, 478691328) 0 0 00000080:00200000:0.0:1788201706.668569:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.668572:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116867, 116867] 00020000:00200000:0.0:1788201706.668574:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478687232 chunk: [478687232, 478691328) 478691328, 4096 00020000:00200000:0.0:1788201706.668575:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.668581:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116867, 116867] 00000008:00010000:0.0:1788201706.668584:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.668586:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.668590:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668596:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478687232 478691327) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668602:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668610:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668614:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.668618:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478687232, 478691328) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.668621:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116867 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.668624:0:17463:0:(rw26.c:898:ll_write_end()) pos 478687232, len 4096, copied 4096 00000080:00200000:0.0:1788201706.668626:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000acecc61[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116867] 00000080:00200000:0.0:1788201706.668628:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008c882baf fffffc0001029 2:0 ffff93c0c1382ee0 116867 lru 00000080:00200000:0.0:1788201706.668631:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007615e549 116867: 1< 2 - - > 2< 478687232 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68452 - -> 00000080:00200000:0.0:1788201706.668632:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000acecc61 00000080:00200000:0.0:1788201706.668633:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.668634:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.668640:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.668647:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.668649:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.668650:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.668656:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.668658:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.668659:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.668661:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668670:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.668674:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.668676:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.668678:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.668680:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116867, 116867] 00000080:00200000:0.0:1788201706.668686:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.668688:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.668705:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.668708:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.668717:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478691328, count: 4096 00000080:00200000:0.0:1788201706.668723:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478691328, bytes: 4096 00000080:00200000:0.0:1788201706.668725:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.668727:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.668728:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478691328, 478695424) 0 0 00000080:00200000:0.0:1788201706.668730:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.668733:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116868, 116868] 00020000:00200000:0.0:1788201706.668734:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478691328 chunk: [478691328, 478695424) 478695424, 4096 00020000:00200000:0.0:1788201706.668736:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.668742:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116868, 116868] 00000008:00010000:0.0:1788201706.668745:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.668747:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.668751:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668757:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478691328 478695423) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668766:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668770:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668774:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.668778:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478691328, 478695424) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.668781:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116868 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.668785:0:17463:0:(rw26.c:898:ll_write_end()) pos 478691328, len 4096, copied 4096 00000080:00200000:0.0:1788201706.668787:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000baaf9a95[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116868] 00000080:00200000:0.0:1788201706.668789:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006668a1fa fffffc0001029 2:0 ffff93c0c13827e0 116868 lru 00000080:00200000:0.0:1788201706.668793:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004d6a39a4 116868: 1< 2 - - > 2< 478691328 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68453 - -> 00000080:00200000:0.0:1788201706.668794:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000baaf9a95 00000080:00200000:0.0:1788201706.668795:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.668796:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.668802:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.668809:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.668811:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.668812:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.668818:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.668819:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.668821:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.668823:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668845:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.668851:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.668852:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.668855:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.668857:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116868, 116868] 00000080:00200000:0.0:1788201706.668866:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.668868:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.668888:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.668890:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.668900:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478695424, count: 4096 00000080:00200000:0.0:1788201706.668906:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478695424, bytes: 4096 00000080:00200000:0.0:1788201706.668907:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.668910:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.668911:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478695424, 478699520) 0 0 00000080:00200000:0.0:1788201706.668912:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.668915:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116869, 116869] 00020000:00200000:0.0:1788201706.668917:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478695424 chunk: [478695424, 478699520) 478699520, 4096 00020000:00200000:0.0:1788201706.668919:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.668925:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116869, 116869] 00000008:00010000:0.0:1788201706.668928:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.668929:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.668934:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668943:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478695424 478699519) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668948:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668953:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.668957:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.668961:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478695424, 478699520) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.668964:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116869 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.668968:0:17463:0:(rw26.c:898:ll_write_end()) pos 478695424, len 4096, copied 4096 00000080:00200000:0.0:1788201706.668971:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000016e1f69a[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116869] 00000080:00200000:0.0:1788201706.668973:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c8dc850a fffffc0001029 2:0 ffff93c0c1383e00 116869 lru 00000080:00200000:0.0:1788201706.668976:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cbdc1c27 116869: 1< 2 - - > 2< 478695424 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68454 - -> 00000080:00200000:0.0:1788201706.668978:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000016e1f69a 00000080:00200000:0.0:1788201706.668979:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.668980:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.668985:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.668993:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.668994:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.668996:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.669002:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.669003:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.669005:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.669007:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669012:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.669033:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.669034:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.669037:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.669039:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116869, 116869] 00000080:00200000:0.0:1788201706.669046:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.669048:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.669067:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.669069:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.669079:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478699520, count: 4096 00000080:00200000:0.0:1788201706.669085:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478699520, bytes: 4096 00000080:00200000:0.0:1788201706.669087:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.669089:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.669090:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478699520, 478703616) 0 0 00000080:00200000:0.0:1788201706.669092:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.669095:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116870, 116870] 00020000:00200000:0.0:1788201706.669097:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478699520 chunk: [478699520, 478703616) 478703616, 4096 00020000:00200000:0.0:1788201706.669098:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.669105:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116870, 116870] 00000008:00010000:0.0:1788201706.669108:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.669109:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.669114:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669120:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478699520 478703615) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669126:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669130:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669134:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.669138:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478699520, 478703616) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.669141:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116870 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.669144:0:17463:0:(rw26.c:898:ll_write_end()) pos 478699520, len 4096, copied 4096 00000080:00200000:0.0:1788201706.669146:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000040bbff6a[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116870] 00000080:00200000:0.0:1788201706.669148:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000489e70b4 fffffc0001029 2:0 ffff93c0c13831c0 116870 lru 00000080:00200000:0.0:1788201706.669152:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000876552a9 116870: 1< 2 - - > 2< 478699520 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68455 - -> 00000080:00200000:0.0:1788201706.669153:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000040bbff6a 00000080:00200000:0.0:1788201706.669154:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.669155:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.669160:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.669168:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.669170:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.669171:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.669177:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.669179:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.669181:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.669183:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669187:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.669192:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.669193:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.669196:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.669198:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116870, 116870] 00000080:00200000:0.0:1788201706.669204:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.669206:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.669231:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.669235:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.669251:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478703616, count: 4096 00000080:00200000:0.0:1788201706.669262:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478703616, bytes: 4096 00000080:00200000:0.0:1788201706.669265:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.669268:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.669270:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478703616, 478707712) 0 0 00000080:00200000:0.0:1788201706.669272:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.669277:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116871, 116871] 00020000:00200000:0.0:1788201706.669280:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478703616 chunk: [478703616, 478707712) 478707712, 4096 00020000:00200000:0.0:1788201706.669281:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.669292:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116871, 116871] 00000008:00010000:0.0:1788201706.669296:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.669299:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.669305:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669314:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478703616 478707711) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669323:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669330:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669337:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.669345:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478703616, 478707712) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.669348:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116871 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.669352:0:17463:0:(rw26.c:898:ll_write_end()) pos 478703616, len 4096, copied 4096 00000080:00200000:0.0:1788201706.669355:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f750fcf2[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116871] 00000080:00200000:0.0:1788201706.669357:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ecc6432a fffffc0001029 2:0 ffff93c0c1383620 116871 lru 00000080:00200000:0.0:1788201706.669360:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001de3778e 116871: 1< 2 - - > 2< 478703616 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68456 - -> 00000080:00200000:0.0:1788201706.669361:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f750fcf2 00000080:00200000:0.0:1788201706.669362:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.669363:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.669369:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.669380:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.669382:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.669383:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.669390:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.669391:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.669393:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.669395:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669400:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.669405:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.669406:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.669409:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.669411:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116871, 116871] 00000080:00200000:0.0:1788201706.669418:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.669420:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.669438:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.669440:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.669451:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478707712, count: 4096 00000080:00200000:0.0:1788201706.669457:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478707712, bytes: 4096 00000080:00200000:0.0:1788201706.669458:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.669461:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.669462:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478707712, 478711808) 0 0 00000080:00200000:0.0:1788201706.669463:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.669466:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116872, 116872] 00020000:00200000:0.0:1788201706.669469:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478707712 chunk: [478707712, 478711808) 478711808, 4096 00020000:00200000:0.0:1788201706.669470:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.669476:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116872, 116872] 00000008:00010000:0.0:1788201706.669480:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.669481:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.669486:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669492:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478707712 478711807) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669497:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669501:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669505:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.669510:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478707712, 478711808) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.669513:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116872 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.669517:0:17463:0:(rw26.c:898:ll_write_end()) pos 478707712, len 4096, copied 4096 00000080:00200000:0.0:1788201706.669519:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000acbb72d6[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116872] 00000080:00200000:0.0:1788201706.669521:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a23c6169 fffffc0001029 2:0 ffff93c0c1383d20 116872 lru 00000080:00200000:0.0:1788201706.669524:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e5a907ca 116872: 1< 2 - - > 2< 478707712 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68457 - -> 00000080:00200000:0.0:1788201706.669526:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000acbb72d6 00000080:00200000:0.0:1788201706.669527:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.669528:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.669533:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.669541:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.669543:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.669544:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.669550:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.669552:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.669553:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.669555:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669560:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.669565:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.669566:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.669568:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.669570:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116872, 116872] 00000080:00200000:0.0:1788201706.669577:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.669579:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.669596:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.669598:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.669608:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478711808, count: 4096 00000080:00200000:0.0:1788201706.669614:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478711808, bytes: 4096 00000080:00200000:0.0:1788201706.669616:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.669618:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.669619:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478711808, 478715904) 0 0 00000080:00200000:0.0:1788201706.669621:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.669624:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116873, 116873] 00020000:00200000:0.0:1788201706.669625:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478711808 chunk: [478711808, 478715904) 478715904, 4096 00020000:00200000:0.0:1788201706.669627:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.669633:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116873, 116873] 00000008:00010000:0.0:1788201706.669636:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.669638:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.669642:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669648:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478711808 478715903) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669653:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669658:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669662:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.669666:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478711808, 478715904) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.669668:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116873 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.669672:0:17463:0:(rw26.c:898:ll_write_end()) pos 478711808, len 4096, copied 4096 00000080:00200000:0.0:1788201706.669674:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ceb4bcee[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116873] 00000080:00200000:0.0:1788201706.669675:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002c719383 fffffc0001029 2:0 ffff93c0c13839a0 116873 lru 00000080:00200000:0.0:1788201706.669679:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000064e05af 116873: 1< 2 - - > 2< 478711808 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68458 - -> 00000080:00200000:0.0:1788201706.669682:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ceb4bcee 00000080:00200000:0.0:1788201706.669682:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.669684:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.669689:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.669696:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.669698:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.669699:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.669705:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.669707:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.669708:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.669710:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669715:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.669719:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.669721:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.669724:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.669726:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116873, 116873] 00000080:00200000:0.0:1788201706.669732:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.669734:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.669755:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.669759:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.669775:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478715904, count: 4096 00000080:00200000:0.0:1788201706.669786:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478715904, bytes: 4096 00000080:00200000:0.0:1788201706.669789:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.669792:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.669794:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478715904, 478720000) 0 0 00000080:00200000:0.0:1788201706.669796:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.669801:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116874, 116874] 00020000:00200000:0.0:1788201706.669803:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478715904 chunk: [478715904, 478720000) 478720000, 4096 00020000:00200000:0.0:1788201706.669806:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.669818:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116874, 116874] 00000008:00010000:0.0:1788201706.669823:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.669825:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.669849:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669859:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478715904 478719999) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669868:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669876:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669883:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.669890:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478715904, 478720000) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.669896:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116874 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.669901:0:17463:0:(rw26.c:898:ll_write_end()) pos 478715904, len 4096, copied 4096 00000080:00200000:0.0:1788201706.669905:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006ae6a522[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116874] 00000080:00200000:0.0:1788201706.669908:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bb87ebb1 fffffc0001029 2:0 ffff93c0c1383b60 116874 lru 00000080:00200000:0.0:1788201706.669913:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001d013190 116874: 1< 2 - - > 2< 478715904 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68459 - -> 00000080:00200000:0.0:1788201706.669915:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006ae6a522 00000080:00200000:0.0:1788201706.669916:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.669918:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.669927:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.669939:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.669942:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.669944:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.669953:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.669956:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.669958:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.669962:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.669970:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.669978:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.669980:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.669985:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.669988:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116874, 116874] 00000080:00200000:0.0:1788201706.669999:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.670002:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.670047:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.670051:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.670069:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478720000, count: 4096 00000080:00200000:0.0:1788201706.670080:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478720000, bytes: 4096 00000080:00200000:0.0:1788201706.670083:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.670085:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.670086:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478720000, 478724096) 0 0 00000080:00200000:0.0:1788201706.670088:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.670092:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116875, 116875] 00020000:00200000:0.0:1788201706.670094:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478720000 chunk: [478720000, 478724096) 478724096, 4096 00020000:00200000:0.0:1788201706.670095:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.670103:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116875, 116875] 00000008:00010000:0.0:1788201706.670107:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.670108:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.670114:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670120:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478720000 478724095) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670125:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670130:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670134:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.670138:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478720000, 478724096) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.670142:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116875 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.670147:0:17463:0:(rw26.c:898:ll_write_end()) pos 478720000, len 4096, copied 4096 00000080:00200000:0.0:1788201706.670151:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005bec4be3[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116875] 00000080:00200000:0.0:1788201706.670154:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000064e14798 fffffc0001029 2:0 ffff93c0c1383c40 116875 lru 00000080:00200000:0.0:1788201706.670159:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000044075c03 116875: 1< 2 - - > 2< 478720000 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68460 - -> 00000080:00200000:0.0:1788201706.670161:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005bec4be3 00000080:00200000:0.0:1788201706.670162:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.670164:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.670173:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.670185:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.670188:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.670190:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.670199:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.670202:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.670204:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.670209:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670217:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.670226:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.670228:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.670232:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.670236:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116875, 116875] 00000080:00200000:0.0:1788201706.670247:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.670251:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.670277:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.670280:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.670291:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478724096, count: 4096 00000080:00200000:0.0:1788201706.670297:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478724096, bytes: 4096 00000080:00200000:0.0:1788201706.670299:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.670301:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.670303:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478724096, 478728192) 0 0 00000080:00200000:0.0:1788201706.670304:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.670307:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116876, 116876] 00020000:00200000:0.0:1788201706.670310:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478724096 chunk: [478724096, 478728192) 478728192, 4096 00020000:00200000:0.0:1788201706.670312:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.670318:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116876, 116876] 00000008:00010000:0.0:1788201706.670322:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.670323:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.670329:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670334:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478724096 478728191) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670340:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670345:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670349:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.670353:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478724096, 478728192) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.670357:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116876 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.670361:0:17463:0:(rw26.c:898:ll_write_end()) pos 478724096, len 4096, copied 4096 00000080:00200000:0.0:1788201706.670363:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006528cf46[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116876] 00000080:00200000:0.0:1788201706.670365:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001c1286ac fffffc0001029 2:0 ffff93c0c1383540 116876 lru 00000080:00200000:0.0:1788201706.670369:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008739f78a 116876: 1< 2 - - > 2< 478724096 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68461 - -> 00000080:00200000:0.0:1788201706.670370:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006528cf46 00000080:00200000:0.0:1788201706.670371:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.670372:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.670378:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.670386:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.670387:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.670389:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.670395:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.670397:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.670398:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.670400:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670405:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.670410:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.670411:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.670414:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.670416:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116876, 116876] 00000080:00200000:0.0:1788201706.670423:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.670424:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.670442:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.670444:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.670455:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478728192, count: 4096 00000080:00200000:0.0:1788201706.670461:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478728192, bytes: 4096 00000080:00200000:0.0:1788201706.670462:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.670464:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.670465:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478728192, 478732288) 0 0 00000080:00200000:0.0:1788201706.670467:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.670470:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116877, 116877] 00020000:00200000:0.0:1788201706.670472:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478728192 chunk: [478728192, 478732288) 478732288, 4096 00020000:00200000:0.0:1788201706.670474:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.670480:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116877, 116877] 00000008:00010000:0.0:1788201706.670483:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.670484:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.670489:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670495:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478728192 478732287) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670500:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670505:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670509:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.670517:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478728192, 478732288) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.670520:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116877 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.670523:0:17463:0:(rw26.c:898:ll_write_end()) pos 478728192, len 4096, copied 4096 00000080:00200000:0.0:1788201706.670526:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004ef64e12[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116877] 00000080:00200000:0.0:1788201706.670528:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003fe0205b fffffc0001029 2:0 ffff93c0c1383460 116877 lru 00000080:00200000:0.0:1788201706.670531:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000cbb2c70 116877: 1< 2 - - > 2< 478728192 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68462 - -> 00000080:00200000:0.0:1788201706.670532:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004ef64e12 00000080:00200000:0.0:1788201706.670533:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.670534:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.670542:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.670553:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.670556:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.670558:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.670567:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.670569:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.670572:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.670576:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670583:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.670591:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.670593:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.670598:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.670601:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116877, 116877] 00000080:00200000:0.0:1788201706.670611:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.670615:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.670646:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.670649:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.670666:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478732288, count: 4096 00000080:00200000:0.0:1788201706.670675:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478732288, bytes: 4096 00000080:00200000:0.0:1788201706.670677:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.670680:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.670681:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478732288, 478736384) 0 0 00000080:00200000:0.0:1788201706.670682:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.670686:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116878, 116878] 00020000:00200000:0.0:1788201706.670688:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478732288 chunk: [478732288, 478736384) 478736384, 4096 00020000:00200000:0.0:1788201706.670690:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.670696:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116878, 116878] 00000008:00010000:0.0:1788201706.670700:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.670701:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.670707:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670712:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478732288 478736383) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670719:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670726:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670738:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.670746:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478732288, 478736384) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.670751:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116878 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.670756:0:17463:0:(rw26.c:898:ll_write_end()) pos 478732288, len 4096, copied 4096 00000080:00200000:0.0:1788201706.670759:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001b2e8947[3 0000000075f4bf38 2 1 000000009c1bb794 0 0 116878] 00000080:00200000:0.0:1788201706.670762:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c8816814 fffffc0001029 2:0 ffff93c0c13830e0 116878 lru 00000080:00200000:0.0:1788201706.670767:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a467c3ce 116878: 1< 2 - - > 2< 478732288 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68463 - -> 00000080:00200000:0.0:1788201706.670768:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001b2e8947 00000080:00200000:0.0:1788201706.670769:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.670771:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.670779:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.670791:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.670794:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.670795:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.670804:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.670807:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.670809:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.670812:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.670819:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.670854:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02e48 (ffff93c1c85e7ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.670856:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.670862:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.670865:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116878, 116878] 00000080:00200000:0.0:1788201706.670881:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.670884:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.670935:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.670941:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.670969:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478736384, count: 4096 00000080:00200000:1.0:1788201706.670998:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478736384, bytes: 4096 00000080:00200000:1.0:1788201706.671003:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.671007:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.671009:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478736384, 478740480) 0 0 00000080:00200000:1.0:1788201706.671012:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.671031:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116879, 116879] 00020000:00200000:1.0:1788201706.671035:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478736384 chunk: [478736384, 478740480) 478740480, 4096 00020000:00200000:1.0:1788201706.671038:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.671055:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116879, 116879] 00000008:00010000:1.0:1788201706.671060:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.671063:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.671072:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671083:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478736384 478740479) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671093:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671102:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671109:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.671118:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478736384, 478740480) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.671124:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116879 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.671131:0:17463:0:(rw26.c:898:ll_write_end()) pos 478736384, len 4096, copied 4096 00000080:00200000:1.0:1788201706.671136:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ced54203[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116879] 00000080:00200000:1.0:1788201706.671139:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001863faa5 fffffc0001029 2:0 ffff93c0c1383700 116879 lru 00000080:00200000:1.0:1788201706.671146:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d0056919 116879: 1< 2 - - > 2< 478736384 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68464 - -> 00000080:00200000:1.0:1788201706.671148:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ced54203 00000080:00200000:1.0:1788201706.671149:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.671151:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.671164:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.671178:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.671181:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.671183:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.671196:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.671199:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.671202:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.671206:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671224:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.671233:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.671236:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.671241:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.671244:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116879, 116879] 00000080:00200000:1.0:1788201706.671260:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.671264:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.671299:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.671303:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.671321:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478740480, count: 4096 00000080:00200000:1.0:1788201706.671332:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478740480, bytes: 4096 00000080:00200000:1.0:1788201706.671335:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.671339:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.671341:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478740480, 478744576) 0 0 00000080:00200000:1.0:1788201706.671343:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.671349:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116880, 116880] 00020000:00200000:1.0:1788201706.671352:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478740480 chunk: [478740480, 478744576) 478744576, 4096 00020000:00200000:1.0:1788201706.671355:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.671367:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116880, 116880] 00000008:00010000:1.0:1788201706.671373:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.671375:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.671384:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671393:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478740480 478744575) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671404:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671412:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671419:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.671427:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478740480, 478744576) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.671434:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116880 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.671440:0:17463:0:(rw26.c:898:ll_write_end()) pos 478740480, len 4096, copied 4096 00000080:00200000:1.0:1788201706.671445:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000052d0cf27[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116880] 00000080:00200000:1.0:1788201706.671448:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000075abd799 fffffc0001029 2:0 ffff93c0c1383000 116880 lru 00000080:00200000:1.0:1788201706.671454:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ace6e429 116880: 1< 2 - - > 2< 478740480 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68465 - -> 00000080:00200000:1.0:1788201706.671456:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000052d0cf27 00000080:00200000:1.0:1788201706.671458:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.671460:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.671469:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.671481:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.671485:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.671487:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.671498:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.671500:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.671503:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.671507:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671516:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.671523:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.671526:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.671529:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.671533:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116880, 116880] 00000080:00200000:1.0:1788201706.671544:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.671547:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.671574:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.671577:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.671593:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478744576, count: 4096 00000080:00200000:1.0:1788201706.671605:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478744576, bytes: 4096 00000080:00200000:1.0:1788201706.671608:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.671612:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.671614:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478744576, 478748672) 0 0 00000080:00200000:1.0:1788201706.671616:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.671621:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116881, 116881] 00020000:00200000:1.0:1788201706.671624:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478744576 chunk: [478744576, 478748672) 478748672, 4096 00020000:00200000:1.0:1788201706.671627:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.671639:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116881, 116881] 00000008:00010000:1.0:1788201706.671644:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.671647:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.671656:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671665:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478744576 478748671) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671675:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671683:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671690:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.671703:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478744576, 478748672) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.671708:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116881 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.671713:0:17463:0:(rw26.c:898:ll_write_end()) pos 478744576, len 4096, copied 4096 00000080:00200000:1.0:1788201706.671717:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005aa161d2[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116881] 00000080:00200000:1.0:1788201706.671721:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000eef3e0d6 fffffc0001029 2:0 ffff93c0c1383a80 116881 lru 00000080:00200000:1.0:1788201706.671727:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b44ba002 116881: 1< 2 - - > 2< 478744576 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68466 - -> 00000080:00200000:1.0:1788201706.671729:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005aa161d2 00000080:00200000:1.0:1788201706.671730:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.671732:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.671741:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.671752:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.671755:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.671757:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.671766:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.671769:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.671771:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.671775:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671783:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.671791:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.671794:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.671798:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.671801:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116881, 116881] 00000080:00200000:1.0:1788201706.671812:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.671815:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.671855:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.671858:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.671870:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478748672, count: 4096 00000080:00200000:1.0:1788201706.671876:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478748672, bytes: 4096 00000080:00200000:1.0:1788201706.671878:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.671881:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.671882:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478748672, 478752768) 0 0 00000080:00200000:1.0:1788201706.671884:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.671887:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116882, 116882] 00020000:00200000:1.0:1788201706.671889:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478748672 chunk: [478748672, 478752768) 478752768, 4096 00020000:00200000:1.0:1788201706.671891:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.671898:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116882, 116882] 00000008:00010000:1.0:1788201706.671902:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.671903:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.671909:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671915:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478748672 478752767) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671920:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671925:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671933:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.671937:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478748672, 478752768) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.671940:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116882 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.671944:0:17463:0:(rw26.c:898:ll_write_end()) pos 478748672, len 4096, copied 4096 00000080:00200000:1.0:1788201706.671947:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000075f89dda[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116882] 00000080:00200000:1.0:1788201706.671948:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fd138149 fffffc0001029 2:0 ffff93c0c1383380 116882 lru 00000080:00200000:1.0:1788201706.671952:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000210e64e0 116882: 1< 2 - - > 2< 478748672 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68467 - -> 00000080:00200000:1.0:1788201706.671954:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000075f89dda 00000080:00200000:1.0:1788201706.671954:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.671956:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.671962:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.671970:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.671972:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.671973:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.671979:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.671981:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.671983:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.671985:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.671990:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.671999:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.672001:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.672003:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.672005:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116882, 116882] 00000080:00200000:1.0:1788201706.672013:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.672028:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.672052:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.672056:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.672071:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478752768, count: 4096 00000080:00200000:1.0:1788201706.672082:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478752768, bytes: 4096 00000080:00200000:1.0:1788201706.672085:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.672089:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.672091:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478752768, 478756864) 0 0 00000080:00200000:1.0:1788201706.672093:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.672099:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116883, 116883] 00020000:00200000:1.0:1788201706.672101:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478752768 chunk: [478752768, 478756864) 478756864, 4096 00020000:00200000:1.0:1788201706.672102:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.672111:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116883, 116883] 00000008:00010000:1.0:1788201706.672114:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.672116:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.672122:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672128:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478752768 478756863) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672133:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672142:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672146:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.672150:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478752768, 478756864) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.672153:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116883 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.672157:0:17463:0:(rw26.c:898:ll_write_end()) pos 478752768, len 4096, copied 4096 00000080:00200000:1.0:1788201706.672160:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000080d645ea[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116883] 00000080:00200000:1.0:1788201706.672161:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005d66bee3 fffffc0001029 2:0 ffff93c0c13838c0 116883 lru 00000080:00200000:1.0:1788201706.672165:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c5be0a08 116883: 1< 2 - - > 2< 478752768 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68468 - -> 00000080:00200000:1.0:1788201706.672166:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000080d645ea 00000080:00200000:1.0:1788201706.672167:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.672168:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.672174:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.672182:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.672184:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.672185:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.672192:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.672194:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.672196:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.672199:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672212:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.672220:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.672223:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.672227:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.672230:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116883, 116883] 00000080:00200000:1.0:1788201706.672241:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.672244:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.672271:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.672274:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.672289:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478756864, count: 4096 00000080:00200000:1.0:1788201706.672298:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478756864, bytes: 4096 00000080:00200000:1.0:1788201706.672301:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.672305:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.672306:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478756864, 478760960) 0 0 00000080:00200000:1.0:1788201706.672309:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.672314:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116884, 116884] 00020000:00200000:1.0:1788201706.672316:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478756864 chunk: [478756864, 478760960) 478760960, 4096 00020000:00200000:1.0:1788201706.672319:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.672329:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116884, 116884] 00000008:00010000:1.0:1788201706.672334:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.672336:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.672342:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672348:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478756864 478760959) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672357:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672361:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672365:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.672370:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478756864, 478760960) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.672374:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116884 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.672378:0:17463:0:(rw26.c:898:ll_write_end()) pos 478756864, len 4096, copied 4096 00000080:00200000:1.0:1788201706.672380:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000029def88[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116884] 00000080:00200000:1.0:1788201706.672382:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000056d08ac9 fffffc0001029 2:0 ffff93c0c1383ee0 116884 lru 00000080:00200000:1.0:1788201706.672386:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e283baa7 116884: 1< 2 - - > 2< 478756864 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68469 - -> 00000080:00200000:1.0:1788201706.672388:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000029def88 00000080:00200000:1.0:1788201706.672388:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.672390:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.672396:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.672404:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.672406:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.672407:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.672414:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.672415:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.672417:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.672420:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672425:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.672430:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.672431:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.672433:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.672435:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116884, 116884] 00000080:00200000:1.0:1788201706.672443:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.672445:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.672462:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.672465:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.672475:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478760960, count: 4096 00000080:00200000:1.0:1788201706.672481:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478760960, bytes: 4096 00000080:00200000:1.0:1788201706.672483:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.672485:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.672486:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478760960, 478765056) 0 0 00000080:00200000:1.0:1788201706.672488:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.672491:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116885, 116885] 00020000:00200000:1.0:1788201706.672493:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478760960 chunk: [478760960, 478765056) 478765056, 4096 00020000:00200000:1.0:1788201706.672494:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.672501:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116885, 116885] 00000008:00010000:1.0:1788201706.672504:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.672506:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.672510:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672520:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478760960 478765055) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672525:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672530:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672534:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.672538:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478760960, 478765056) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.672542:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116885 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.672546:0:17463:0:(rw26.c:898:ll_write_end()) pos 478760960, len 4096, copied 4096 00000080:00200000:1.0:1788201706.672548:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d95f6a66[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116885] 00000080:00200000:1.0:1788201706.672550:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000df5f2240 fffffc0001029 2:0 ffff93c0c13837e0 116885 lru 00000080:00200000:1.0:1788201706.672554:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000035e4fdfd 116885: 1< 2 - - > 2< 478760960 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68470 - -> 00000080:00200000:1.0:1788201706.672555:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d95f6a66 00000080:00200000:1.0:1788201706.672556:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.672557:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.672563:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.672570:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.672572:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.672574:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.672580:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.672581:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.672583:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.672585:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672590:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.672595:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.672597:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.672599:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.672601:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116885, 116885] 00000080:00200000:1.0:1788201706.672608:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.672609:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.672627:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.672629:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.672639:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478765056, count: 4096 00000080:00200000:1.0:1788201706.672645:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478765056, bytes: 4096 00000080:00200000:1.0:1788201706.672647:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.672649:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.672650:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478765056, 478769152) 0 0 00000080:00200000:1.0:1788201706.672652:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.672655:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116886, 116886] 00020000:00200000:1.0:1788201706.672657:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478765056 chunk: [478765056, 478769152) 478769152, 4096 00020000:00200000:1.0:1788201706.672658:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.672665:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116886, 116886] 00000008:00010000:1.0:1788201706.672668:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.672669:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.672675:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672681:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478765056 478769151) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672686:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672690:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672694:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.672698:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478765056, 478769152) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.672701:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116886 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.672704:0:17463:0:(rw26.c:898:ll_write_end()) pos 478765056, len 4096, copied 4096 00000080:00200000:1.0:1788201706.672707:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a28b78cd[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116886] 00000080:00200000:1.0:1788201706.672708:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000976b3c39 fffffc0001029 2:0 ffff93c0c13832a0 116886 lru 00000080:00200000:1.0:1788201706.672712:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002a16d920 116886: 1< 2 - - > 2< 478765056 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68471 - -> 00000080:00200000:1.0:1788201706.672713:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a28b78cd 00000080:00200000:1.0:1788201706.672714:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.672715:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.672721:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.672728:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.672730:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.672732:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.672737:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.672739:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.672741:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.672743:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672748:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.672752:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.672754:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.672756:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.672758:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116886, 116886] 00000080:00200000:1.0:1788201706.672765:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.672767:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.672784:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.672787:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.672797:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478769152, count: 4096 00000080:00200000:1.0:1788201706.672803:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478769152, bytes: 4096 00000080:00200000:1.0:1788201706.672804:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.672807:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.672808:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478769152, 478773248) 0 0 00000080:00200000:1.0:1788201706.672809:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.672812:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116887, 116887] 00020000:00200000:1.0:1788201706.672814:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478769152 chunk: [478769152, 478773248) 478773248, 4096 00020000:00200000:1.0:1788201706.672815:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.672822:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116887, 116887] 00000008:00010000:1.0:1788201706.672825:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.672827:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.672847:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672854:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478769152 478773247) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672859:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672863:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672867:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.672872:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478769152, 478773248) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.672875:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116887 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.672879:0:17463:0:(rw26.c:898:ll_write_end()) pos 478769152, len 4096, copied 4096 00000080:00200000:1.0:1788201706.672881:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008edaedde[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116887] 00000080:00200000:1.0:1788201706.672883:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000013f82fa6 fffffc0001029 2:0 ffff93c0c1384a80 116887 lru 00000080:00200000:1.0:1788201706.672887:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000026a25d10 116887: 1< 2 - - > 2< 478769152 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68472 - -> 00000080:00200000:1.0:1788201706.672888:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008edaedde 00000080:00200000:1.0:1788201706.672889:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.672891:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.672896:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.672907:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.672909:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.672911:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.672917:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.672918:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.672920:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.672922:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.672927:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.672933:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.672934:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.672936:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.672938:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116887, 116887] 00000080:00200000:1.0:1788201706.672945:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.672947:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.672965:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.672967:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.672977:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478773248, count: 4096 00000080:00200000:1.0:1788201706.672983:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478773248, bytes: 4096 00000080:00200000:1.0:1788201706.672985:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.672987:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.672988:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478773248, 478777344) 0 0 00000080:00200000:1.0:1788201706.672989:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.672992:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116888, 116888] 00020000:00200000:1.0:1788201706.672995:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478773248 chunk: [478773248, 478777344) 478777344, 4096 00020000:00200000:1.0:1788201706.672996:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.673003:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116888, 116888] 00000008:00010000:1.0:1788201706.673005:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.673007:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.673012:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673032:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478773248 478777343) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673038:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673044:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673051:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.673059:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478773248, 478777344) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.673064:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116888 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.673070:0:17463:0:(rw26.c:898:ll_write_end()) pos 478773248, len 4096, copied 4096 00000080:00200000:1.0:1788201706.673074:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d8b26aa9[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116888] 00000080:00200000:1.0:1788201706.673077:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000275c0f55 fffffc0001029 2:0 ffff93c0c1384380 116888 lru 00000080:00200000:1.0:1788201706.673083:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000de49f4e2 116888: 1< 2 - - > 2< 478773248 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68473 - -> 00000080:00200000:1.0:1788201706.673086:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d8b26aa9 00000080:00200000:1.0:1788201706.673087:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.673089:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.673097:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.673106:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.673108:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.673110:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.673117:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.673118:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.673120:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.673122:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673127:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.673132:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.673134:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.673136:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.673138:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116888, 116888] 00000080:00200000:1.0:1788201706.673145:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.673147:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.673166:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.673168:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.673178:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478777344, count: 4096 00000080:00200000:1.0:1788201706.673185:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478777344, bytes: 4096 00000080:00200000:1.0:1788201706.673186:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.673189:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.673190:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478777344, 478781440) 0 0 00000080:00200000:1.0:1788201706.673191:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.673195:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116889, 116889] 00020000:00200000:1.0:1788201706.673197:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478777344 chunk: [478777344, 478781440) 478781440, 4096 00020000:00200000:1.0:1788201706.673199:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.673210:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116889, 116889] 00000008:00010000:1.0:1788201706.673215:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.673218:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.673225:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673235:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478777344 478781439) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673244:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673251:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673258:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.673265:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478777344, 478781440) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.673269:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116889 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.673274:0:17463:0:(rw26.c:898:ll_write_end()) pos 478777344, len 4096, copied 4096 00000080:00200000:1.0:1788201706.673278:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000079d87e5b[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116889] 00000080:00200000:1.0:1788201706.673280:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002c9630d7 fffffc0001029 2:0 ffff93c0c13848c0 116889 lru 00000080:00200000:1.0:1788201706.673286:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b9986d12 116889: 1< 2 - - > 2< 478777344 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68474 - -> 00000080:00200000:1.0:1788201706.673289:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000079d87e5b 00000080:00200000:1.0:1788201706.673290:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.673292:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.673300:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.673311:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.673314:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.673315:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.673324:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.673326:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.673328:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.673332:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673339:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.673346:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.673348:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.673352:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.673355:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116889, 116889] 00000080:00200000:1.0:1788201706.673365:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.673369:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.673394:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.673397:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.673411:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478781440, count: 4096 00000080:00200000:1.0:1788201706.673421:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478781440, bytes: 4096 00000080:00200000:1.0:1788201706.673424:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.673427:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.673429:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478781440, 478785536) 0 0 00000080:00200000:1.0:1788201706.673432:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.673436:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116890, 116890] 00020000:00200000:1.0:1788201706.673439:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478781440 chunk: [478781440, 478785536) 478785536, 4096 00020000:00200000:1.0:1788201706.673442:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.673453:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116890, 116890] 00000008:00010000:1.0:1788201706.673458:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.673460:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.673467:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673476:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478781440 478785535) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673485:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673493:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673500:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.673507:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478781440, 478785536) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.673512:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116890 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.673517:0:17463:0:(rw26.c:898:ll_write_end()) pos 478781440, len 4096, copied 4096 00000080:00200000:1.0:1788201706.673521:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e0a38435[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116890] 00000080:00200000:1.0:1788201706.673523:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ef486970 fffffc0001029 2:0 ffff93c0c1384ee0 116890 lru 00000080:00200000:1.0:1788201706.673529:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c9a3f14e 116890: 1< 2 - - > 2< 478781440 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68475 - -> 00000080:00200000:1.0:1788201706.673532:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e0a38435 00000080:00200000:1.0:1788201706.673533:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.673535:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.673541:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.673549:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.673551:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.673553:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.673559:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.673561:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.673562:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.673565:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673570:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.673575:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.673576:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.673579:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.673581:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116890, 116890] 00000080:00200000:1.0:1788201706.673588:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.673590:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.673609:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.673611:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.673621:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478785536, count: 4096 00000080:00200000:1.0:1788201706.673627:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478785536, bytes: 4096 00000080:00200000:1.0:1788201706.673629:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.673631:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.673632:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478785536, 478789632) 0 0 00000080:00200000:1.0:1788201706.673634:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.673637:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116891, 116891] 00020000:00200000:1.0:1788201706.673639:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478785536 chunk: [478785536, 478789632) 478789632, 4096 00020000:00200000:1.0:1788201706.673641:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.673647:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116891, 116891] 00000008:00010000:1.0:1788201706.673650:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.673651:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.673657:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673662:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478785536 478789631) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673668:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673672:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673676:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.673681:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478785536, 478789632) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.673684:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116891 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.673689:0:17463:0:(rw26.c:898:ll_write_end()) pos 478785536, len 4096, copied 4096 00000080:00200000:1.0:1788201706.673691:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000007e80ae[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116891] 00000080:00200000:1.0:1788201706.673693:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000074260394 fffffc0001029 2:0 ffff93c0c13847e0 116891 lru 00000080:00200000:1.0:1788201706.673696:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000015b2009c 116891: 1< 2 - - > 2< 478785536 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68476 - -> 00000080:00200000:1.0:1788201706.673698:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000007e80ae 00000080:00200000:1.0:1788201706.673699:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.673700:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.673706:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.673713:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.673715:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.673716:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.673722:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.673724:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.673725:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.673728:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673732:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.673736:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.673738:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.673740:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.673742:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116891, 116891] 00000080:00200000:1.0:1788201706.673749:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.673751:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.673769:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.673771:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.673781:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478789632, count: 4096 00000080:00200000:1.0:1788201706.673787:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478789632, bytes: 4096 00000080:00200000:1.0:1788201706.673789:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.673791:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.673792:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478789632, 478793728) 0 0 00000080:00200000:1.0:1788201706.673794:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.673797:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116892, 116892] 00020000:00200000:1.0:1788201706.673799:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478789632 chunk: [478789632, 478793728) 478793728, 4096 00020000:00200000:1.0:1788201706.673800:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.673806:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116892, 116892] 00000008:00010000:1.0:1788201706.673809:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.673811:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.673815:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673821:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478789632 478793727) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673827:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673845:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673849:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.673854:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478789632, 478793728) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.673858:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116892 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.673862:0:17463:0:(rw26.c:898:ll_write_end()) pos 478789632, len 4096, copied 4096 00000080:00200000:1.0:1788201706.673864:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002a3e1c44[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116892] 00000080:00200000:1.0:1788201706.673866:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000116e305b fffffc0001029 2:0 ffff93c0c13842a0 116892 lru 00000080:00200000:1.0:1788201706.673869:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e98fd506 116892: 1< 2 - - > 2< 478789632 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68477 - -> 00000080:00200000:1.0:1788201706.673871:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002a3e1c44 00000080:00200000:1.0:1788201706.673871:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.673873:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.673878:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.673886:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.673888:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.673889:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.673895:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.673897:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.673899:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.673901:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673906:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.673910:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.673912:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.673914:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.673916:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116892, 116892] 00000080:00200000:1.0:1788201706.673923:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.673925:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.673943:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.673945:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.673955:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478793728, count: 4096 00000080:00200000:1.0:1788201706.673961:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478793728, bytes: 4096 00000080:00200000:1.0:1788201706.673963:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.673965:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.673966:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478793728, 478797824) 0 0 00000080:00200000:1.0:1788201706.673967:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.673971:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116893, 116893] 00020000:00200000:1.0:1788201706.673972:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478793728 chunk: [478793728, 478797824) 478797824, 4096 00020000:00200000:1.0:1788201706.673974:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.673980:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116893, 116893] 00000008:00010000:1.0:1788201706.673983:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.673984:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.673989:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.673995:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478793728 478797823) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674000:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674004:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674008:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.674030:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478793728, 478797824) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.674033:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116893 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.674037:0:17463:0:(rw26.c:898:ll_write_end()) pos 478793728, len 4096, copied 4096 00000080:00200000:1.0:1788201706.674040:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000706545f9[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116893] 00000080:00200000:1.0:1788201706.674042:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e03a1bd9 fffffc0001029 2:0 ffff93c0c1384e00 116893 lru 00000080:00200000:1.0:1788201706.674048:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c7022a44 116893: 1< 2 - - > 2< 478793728 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68478 - -> 00000080:00200000:1.0:1788201706.674050:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000706545f9 00000080:00200000:1.0:1788201706.674052:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.674053:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.674062:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.674073:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.674076:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.674078:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.674087:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.674090:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.674093:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.674096:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674102:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.674107:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.674108:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.674112:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.674114:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116893, 116893] 00000080:00200000:1.0:1788201706.674122:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.674124:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.674143:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.674145:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.674156:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478797824, count: 4096 00000080:00200000:1.0:1788201706.674162:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478797824, bytes: 4096 00000080:00200000:1.0:1788201706.674163:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.674166:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.674167:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478797824, 478801920) 0 0 00000080:00200000:1.0:1788201706.674168:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.674172:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116894, 116894] 00020000:00200000:1.0:1788201706.674174:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478797824 chunk: [478797824, 478801920) 478801920, 4096 00020000:00200000:1.0:1788201706.674175:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.674182:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116894, 116894] 00000008:00010000:1.0:1788201706.674185:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.674186:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.674192:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674198:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478797824 478801919) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674207:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674215:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674227:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.674236:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478797824, 478801920) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.674241:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116894 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.674246:0:17463:0:(rw26.c:898:ll_write_end()) pos 478797824, len 4096, copied 4096 00000080:00200000:1.0:1788201706.674250:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a1e6287f[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116894] 00000080:00200000:1.0:1788201706.674253:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004c63faaa fffffc0001029 2:0 ffff93c0c13841c0 116894 lru 00000080:00200000:1.0:1788201706.674259:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e7cb45bb 116894: 1< 2 - - > 2< 478797824 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68479 - -> 00000080:00200000:1.0:1788201706.674261:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a1e6287f 00000080:00200000:1.0:1788201706.674262:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.674264:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.674272:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.674281:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.674283:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.674286:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.674295:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.674298:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.674300:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.674304:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674312:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.674326:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.674329:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.674333:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.674336:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116894, 116894] 00000080:00200000:1.0:1788201706.674344:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.674346:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.674365:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.674367:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.674378:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478801920, count: 4096 00000080:00200000:1.0:1788201706.674384:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478801920, bytes: 4096 00000080:00200000:1.0:1788201706.674386:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.674388:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.674389:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478801920, 478806016) 0 0 00000080:00200000:1.0:1788201706.674391:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.674394:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116895, 116895] 00020000:00200000:1.0:1788201706.674396:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478801920 chunk: [478801920, 478806016) 478806016, 4096 00020000:00200000:1.0:1788201706.674397:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.674404:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116895, 116895] 00000008:00010000:1.0:1788201706.674407:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.674409:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.674414:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674420:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478801920 478806015) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674426:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674434:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674438:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.674442:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478801920, 478806016) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.674445:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116895 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.674449:0:17463:0:(rw26.c:898:ll_write_end()) pos 478801920, len 4096, copied 4096 00000080:00200000:1.0:1788201706.674451:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ada31ebf[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116895] 00000080:00200000:1.0:1788201706.674453:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003a52fba6 fffffc0001029 2:0 ffff93c0c1384620 116895 lru 00000080:00200000:1.0:1788201706.674456:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000080cc7015 116895: 1< 2 - - > 2< 478801920 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68480 - -> 00000080:00200000:1.0:1788201706.674457:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ada31ebf 00000080:00200000:1.0:1788201706.674458:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.674459:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.674465:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.674473:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.674475:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.674476:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.674482:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.674484:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.674485:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.674488:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674495:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.674500:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.674502:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.674504:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.674506:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116895, 116895] 00000080:00200000:1.0:1788201706.674513:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.674515:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.674532:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.674534:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.674544:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478806016, count: 4096 00000080:00200000:1.0:1788201706.674550:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478806016, bytes: 4096 00000080:00200000:1.0:1788201706.674552:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.674554:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.674555:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478806016, 478810112) 0 0 00000080:00200000:1.0:1788201706.674557:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.674560:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116896, 116896] 00020000:00200000:1.0:1788201706.674562:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478806016 chunk: [478806016, 478810112) 478810112, 4096 00020000:00200000:1.0:1788201706.674563:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.674569:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116896, 116896] 00000008:00010000:1.0:1788201706.674572:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.674574:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.674578:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674584:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478806016 478810111) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674593:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674597:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674601:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.674606:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478806016, 478810112) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.674609:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116896 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.674612:0:17463:0:(rw26.c:898:ll_write_end()) pos 478806016, len 4096, copied 4096 00000080:00200000:1.0:1788201706.674615:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fc1c0dff[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116896] 00000080:00200000:1.0:1788201706.674616:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000025db74d4 fffffc0001029 2:0 ffff93c0c1384d20 116896 lru 00000080:00200000:1.0:1788201706.674620:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ee800515 116896: 1< 2 - - > 2< 478806016 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68481 - -> 00000080:00200000:1.0:1788201706.674621:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fc1c0dff 00000080:00200000:1.0:1788201706.674622:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.674623:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.674629:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.674637:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.674638:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.674639:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.674645:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.674647:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.674649:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.674651:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674656:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.674660:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.674662:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.674665:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.674666:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116896, 116896] 00000080:00200000:1.0:1788201706.674673:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.674675:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.674693:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.674695:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.674705:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478810112, count: 4096 00000080:00200000:1.0:1788201706.674711:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478810112, bytes: 4096 00000080:00200000:1.0:1788201706.674713:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.674715:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.674716:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478810112, 478814208) 0 0 00000080:00200000:1.0:1788201706.674717:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.674720:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116897, 116897] 00020000:00200000:1.0:1788201706.674722:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478810112 chunk: [478810112, 478814208) 478814208, 4096 00020000:00200000:1.0:1788201706.674723:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.674730:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116897, 116897] 00000008:00010000:1.0:1788201706.674733:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.674734:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.674739:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674748:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478810112 478814207) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674753:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674758:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674761:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.674766:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478810112, 478814208) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.674769:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116897 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.674773:0:17463:0:(rw26.c:898:ll_write_end()) pos 478810112, len 4096, copied 4096 00000080:00200000:1.0:1788201706.674775:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d425f065[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116897] 00000080:00200000:1.0:1788201706.674777:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007378d9ad fffffc0001029 2:0 ffff93c0c13849a0 116897 lru 00000080:00200000:1.0:1788201706.674780:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001e18379d 116897: 1< 2 - - > 2< 478810112 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68482 - -> 00000080:00200000:1.0:1788201706.674782:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d425f065 00000080:00200000:1.0:1788201706.674783:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.674784:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.674789:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.674797:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.674800:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.674802:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.674811:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.674813:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.674816:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.674819:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.674827:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.674859:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.674862:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.674867:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.674871:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116897, 116897] 00000080:00200000:1.0:1788201706.674885:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.674888:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.674919:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.674923:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.674939:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478814208, count: 4096 00000080:00200000:1.0:1788201706.674950:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478814208, bytes: 4096 00000080:00200000:1.0:1788201706.674953:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.674957:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.674959:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478814208, 478818304) 0 0 00000080:00200000:1.0:1788201706.674961:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.674967:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116898, 116898] 00020000:00200000:1.0:1788201706.674970:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478814208 chunk: [478814208, 478818304) 478818304, 4096 00020000:00200000:1.0:1788201706.674972:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.674986:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116898, 116898] 00000008:00010000:1.0:1788201706.674992:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.674994:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.675002:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675012:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478814208 478818303) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675036:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675044:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675051:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.675060:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478814208, 478818304) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.675066:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116898 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.675071:0:17463:0:(rw26.c:898:ll_write_end()) pos 478814208, len 4096, copied 4096 00000080:00200000:1.0:1788201706.675075:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000669a6257[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116898] 00000080:00200000:1.0:1788201706.675078:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000096cb9c3d fffffc0001029 2:0 ffff93c0c1384b60 116898 lru 00000080:00200000:1.0:1788201706.675083:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f24287c5 116898: 1< 2 - - > 2< 478814208 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68483 - -> 00000080:00200000:1.0:1788201706.675086:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000669a6257 00000080:00200000:1.0:1788201706.675087:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.675089:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.675098:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.675110:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.675113:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.675115:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.675124:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.675127:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.675130:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.675133:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675141:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.675149:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.675152:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.675156:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.675159:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116898, 116898] 00000080:00200000:1.0:1788201706.675170:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.675174:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.675204:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.675209:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.675226:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478818304, count: 4096 00000080:00200000:1.0:1788201706.675237:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478818304, bytes: 4096 00000080:00200000:1.0:1788201706.675240:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.675244:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.675245:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478818304, 478822400) 0 0 00000080:00200000:1.0:1788201706.675248:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.675253:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116899, 116899] 00020000:00200000:1.0:1788201706.675256:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478818304 chunk: [478818304, 478822400) 478822400, 4096 00020000:00200000:1.0:1788201706.675259:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.675271:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116899, 116899] 00000008:00010000:1.0:1788201706.675277:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.675280:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.675288:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675297:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478818304 478822399) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675306:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675314:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675322:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.675330:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478818304, 478822400) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.675336:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116899 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.675342:0:17463:0:(rw26.c:898:ll_write_end()) pos 478818304, len 4096, copied 4096 00000080:00200000:1.0:1788201706.675346:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000020e8d53a[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116899] 00000080:00200000:1.0:1788201706.675349:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a1b1b223 fffffc0001029 2:0 ffff93c0c1384c40 116899 lru 00000080:00200000:1.0:1788201706.675354:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ba566e07 116899: 1< 2 - - > 2< 478818304 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68484 - -> 00000080:00200000:1.0:1788201706.675357:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000020e8d53a 00000080:00200000:1.0:1788201706.675358:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.675360:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.675369:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.675387:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.675390:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.675393:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.675403:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.675406:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.675409:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.675414:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675422:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.675431:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.675434:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.675438:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.675441:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116899, 116899] 00000080:00200000:1.0:1788201706.675453:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.675456:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.675488:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.675492:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.675509:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478822400, count: 4096 00000080:00200000:1.0:1788201706.675521:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478822400, bytes: 4096 00000080:00200000:1.0:1788201706.675524:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.675528:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.675529:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478822400, 478826496) 0 0 00000080:00200000:1.0:1788201706.675532:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.675537:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116900, 116900] 00020000:00200000:1.0:1788201706.675541:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478822400 chunk: [478822400, 478826496) 478826496, 4096 00020000:00200000:1.0:1788201706.675543:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.675555:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116900, 116900] 00000008:00010000:1.0:1788201706.675561:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.675564:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.675572:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675582:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478822400 478826495) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675591:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675599:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675606:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.675614:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478822400, 478826496) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.675619:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116900 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.675625:0:17463:0:(rw26.c:898:ll_write_end()) pos 478822400, len 4096, copied 4096 00000080:00200000:1.0:1788201706.675629:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000166596fb[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116900] 00000080:00200000:1.0:1788201706.675632:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001d44f1ff fffffc0001029 2:0 ffff93c0c1384540 116900 lru 00000080:00200000:1.0:1788201706.675638:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004094c742 116900: 1< 2 - - > 2< 478822400 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68485 - -> 00000080:00200000:1.0:1788201706.675640:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000166596fb 00000080:00200000:1.0:1788201706.675641:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.675644:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.675654:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.675667:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.675670:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.675672:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.675682:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.675685:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.675687:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.675692:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675700:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.675709:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.675711:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.675716:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.675719:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116900, 116900] 00000080:00200000:1.0:1788201706.675731:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.675734:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.675767:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.675771:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.675788:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478826496, count: 4096 00000080:00200000:1.0:1788201706.675799:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478826496, bytes: 4096 00000080:00200000:1.0:1788201706.675802:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.675805:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.675807:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478826496, 478830592) 0 0 00000080:00200000:1.0:1788201706.675809:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.675815:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116901, 116901] 00020000:00200000:1.0:1788201706.675818:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478826496 chunk: [478826496, 478830592) 478830592, 4096 00020000:00200000:1.0:1788201706.675821:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.675853:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116901, 116901] 00000008:00010000:1.0:1788201706.675859:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.675862:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.675870:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675879:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478826496 478830591) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675889:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675896:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675903:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.675911:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478826496, 478830592) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.675916:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116901 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.675922:0:17463:0:(rw26.c:898:ll_write_end()) pos 478826496, len 4096, copied 4096 00000080:00200000:1.0:1788201706.675926:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c035db4b[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116901] 00000080:00200000:1.0:1788201706.675929:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bf1cdc51 fffffc0001029 2:0 ffff93c0c1384460 116901 lru 00000080:00200000:1.0:1788201706.675935:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004a77503a 116901: 1< 2 - - > 2< 478826496 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68486 - -> 00000080:00200000:1.0:1788201706.675938:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c035db4b 00000080:00200000:1.0:1788201706.675939:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.675941:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.675950:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.675963:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.675966:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.675968:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.675977:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.675979:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.675982:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.675986:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.675994:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.676002:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.676005:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.676009:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.676012:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116901, 116901] 00000080:00200000:1.0:1788201706.676036:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.676039:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.676069:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.676073:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.676088:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478830592, count: 4096 00000080:00200000:1.0:1788201706.676099:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478830592, bytes: 4096 00000080:00200000:1.0:1788201706.676103:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.676106:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.676108:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478830592, 478834688) 0 0 00000080:00200000:1.0:1788201706.676111:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.676115:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116902, 116902] 00020000:00200000:1.0:1788201706.676118:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478830592 chunk: [478830592, 478834688) 478834688, 4096 00020000:00200000:1.0:1788201706.676121:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.676132:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116902, 116902] 00000008:00010000:1.0:1788201706.676137:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.676140:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.676147:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.676157:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478830592 478834687) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.676166:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.676174:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.676183:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.676190:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478830592, 478834688) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.676195:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116902 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.676201:0:17463:0:(rw26.c:898:ll_write_end()) pos 478830592, len 4096, copied 4096 00000080:00200000:1.0:1788201706.676205:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000db9ae422[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116902] 00000080:00200000:1.0:1788201706.676208:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000918de6ab fffffc0001029 2:0 ffff93c0c13840e0 116902 lru 00000080:00200000:1.0:1788201706.676214:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e9ffd0a5 116902: 1< 2 - - > 2< 478830592 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68487 - -> 00000080:00200000:1.0:1788201706.676216:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000db9ae422 00000080:00200000:1.0:1788201706.676217:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.676220:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.676229:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.676240:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.676243:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.676245:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.676255:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.676258:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.676260:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.676265:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.676272:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.676281:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.676284:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.676288:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.676291:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116902, 116902] 00000080:00200000:1.0:1788201706.676303:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.676306:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.676338:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.676342:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.676358:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478834688, count: 4096 00000080:00200000:1.0:1788201706.676369:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478834688, bytes: 4096 00000080:00200000:1.0:1788201706.676372:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.676376:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.676378:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478834688, 478838784) 0 0 00000080:00200000:1.0:1788201706.676380:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.676386:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116903, 116903] 00020000:00200000:1.0:1788201706.676389:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478834688 chunk: [478834688, 478838784) 478838784, 4096 00020000:00200000:1.0:1788201706.676391:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.676404:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116903, 116903] 00000008:00010000:1.0:1788201706.676409:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.676411:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.676419:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.676428:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478834688 478838783) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.676437:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.676445:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.676452:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.676460:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478834688, 478838784) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.676465:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116903 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.676471:0:17463:0:(rw26.c:898:ll_write_end()) pos 478834688, len 4096, copied 4096 00000080:00200000:1.0:1788201706.676475:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008f8841a8[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116903] 00000080:00200000:1.0:1788201706.676478:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000058e9822a fffffc0001029 2:0 ffff93c0c1384700 116903 lru 00000080:00200000:1.0:1788201706.676483:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ae9ca491 116903: 1< 2 - - > 2< 478834688 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68488 - -> 00000080:00200000:1.0:1788201706.676485:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008f8841a8 00000080:00200000:1.0:1788201706.676486:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.676489:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.676497:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.676509:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.676512:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.676514:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.676523:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.676526:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.676529:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.676533:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.676541:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.676549:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.676552:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.676556:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.676559:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116903, 116903] 00000080:00200000:1.0:1788201706.676570:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.676573:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1788201706.676603:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:1.0:1788201706.676607:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1788201706.676622:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478838784, count: 4096 00000080:00200000:1.0:1788201706.676633:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478838784, bytes: 4096 00000080:00200000:1.0:1788201706.676636:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1788201706.676640:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:1.0:1788201706.676641:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478838784, 478842880) 0 0 00000080:00200000:1.0:1788201706.676644:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788201706.676648:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116904, 116904] 00020000:00200000:1.0:1788201706.676651:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478838784 chunk: [478838784, 478842880) 478842880, 4096 00020000:00200000:1.0:1788201706.676654:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201706.676665:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116904, 116904] 00000008:00010000:1.0:1788201706.676670:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788201706.676672:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c69e7530, osc lock ffff93c1c84c6850, flags 0x0 00010000:00010000:1.0:1788201706.676679:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.676688:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478838784 478842879) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.676698:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.676705:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.676712:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.676720:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478838784, 478842880) append 0 append lockpos 0 00000080:00200000:1.0:1788201706.676726:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116904 of 0 to 4096 bytes 00000080:00200000:1.0:1788201706.676731:0:17463:0:(rw26.c:898:ll_write_end()) pos 478838784, len 4096, copied 4096 00000080:00200000:1.0:1788201706.676735:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005c8c28e7[3 0000000075f4bf38 2 1 00000000554fd9da 0 0 116904] 00000080:00200000:1.0:1788201706.676738:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000076014792 fffffc0001029 2:0 ffff93c0c1384000 116904 lru 00000080:00200000:1.0:1788201706.676744:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d6c4a3df 116904: 1< 2 - - > 2< 478838784 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68489 - -> 00000080:00200000:1.0:1788201706.676747:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005c8c28e7 00000080:00200000:1.0:1788201706.676748:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1788201706.676750:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1788201706.676759:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1788201706.676771:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:1.0:1788201706.676774:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1788201706.676776:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1788201706.676785:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_release():245 00000020:00010000:1.0:1788201706.676788:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788201706.676790:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c69e7530 (ffff93c1d1bf1a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788201706.676794:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1788201706.676803:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1788201706.676811:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d1bee218 (ffff93c1d1bf13f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788201706.676813:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c69e7530 (ffff93c1d1bf13f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788201706.676817:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1788201706.676820:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116904, 116904] 00000080:00200000:1.0:1788201706.676856:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201706.676859:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.676894:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.676899:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.676919:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478842880, count: 4096 00000080:00200000:0.0:1788201706.676933:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478842880, bytes: 4096 00000080:00200000:0.0:1788201706.676935:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.676938:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.676940:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478842880, 478846976) 0 0 00000080:00200000:0.0:1788201706.676941:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.676946:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116905, 116905] 00020000:00200000:0.0:1788201706.676948:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478842880 chunk: [478842880, 478846976) 478846976, 4096 00020000:00200000:0.0:1788201706.676950:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.676962:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116905, 116905] 00000008:00010000:0.0:1788201706.676967:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.676969:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.676976:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.676983:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478842880 478846975) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.676990:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.676999:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677003:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.677008:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478842880, 478846976) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.677012:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116905 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.677029:0:17463:0:(rw26.c:898:ll_write_end()) pos 478842880, len 4096, copied 4096 00000080:00200000:0.0:1788201706.677033:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000019f04de9[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116905] 00000080:00200000:0.0:1788201706.677034:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000015d7031f fffffc0001029 2:0 ffff93c0c1385e00 116905 lru 00000080:00200000:0.0:1788201706.677038:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001737b45b 116905: 1< 2 - - > 2< 478842880 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68490 - -> 00000080:00200000:0.0:1788201706.677039:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000019f04de9 00000080:00200000:0.0:1788201706.677040:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.677042:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.677049:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.677059:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.677061:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.677063:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.677072:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.677074:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.677076:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.677079:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677087:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.677093:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.677094:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.677098:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.677100:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116905, 116905] 00000080:00200000:0.0:1788201706.677111:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.677113:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.677135:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.677137:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.677150:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478846976, count: 4096 00000080:00200000:0.0:1788201706.677156:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478846976, bytes: 4096 00000080:00200000:0.0:1788201706.677158:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.677161:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.677162:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478846976, 478851072) 0 0 00000080:00200000:0.0:1788201706.677163:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.677167:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116906, 116906] 00020000:00200000:0.0:1788201706.677169:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478846976 chunk: [478846976, 478851072) 478851072, 4096 00020000:00200000:0.0:1788201706.677170:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.677178:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116906, 116906] 00000008:00010000:0.0:1788201706.677183:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.677185:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.677193:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677203:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478846976 478851071) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677216:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677224:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677232:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.677240:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478846976, 478851072) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.677244:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116906 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.677249:0:17463:0:(rw26.c:898:ll_write_end()) pos 478846976, len 4096, copied 4096 00000080:00200000:0.0:1788201706.677252:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a8bf42ec[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116906] 00000080:00200000:0.0:1788201706.677254:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000592e76f8 fffffc0001029 2:0 ffff93c0c13851c0 116906 lru 00000080:00200000:0.0:1788201706.677258:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000aec6791b 116906: 1< 2 - - > 2< 478846976 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68491 - -> 00000080:00200000:0.0:1788201706.677259:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a8bf42ec 00000080:00200000:0.0:1788201706.677260:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.677261:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.677268:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.677276:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.677278:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.677279:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.677286:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.677288:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.677290:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.677292:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677298:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.677311:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.677313:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.677316:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.677318:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116906, 116906] 00000080:00200000:0.0:1788201706.677326:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.677328:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.677350:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.677354:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.677370:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478851072, count: 4096 00000080:00200000:0.0:1788201706.677380:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478851072, bytes: 4096 00000080:00200000:0.0:1788201706.677383:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.677387:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.677388:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478851072, 478855168) 0 0 00000080:00200000:0.0:1788201706.677391:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.677395:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116907, 116907] 00020000:00200000:0.0:1788201706.677398:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478851072 chunk: [478851072, 478855168) 478855168, 4096 00020000:00200000:0.0:1788201706.677400:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.677411:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116907, 116907] 00000008:00010000:0.0:1788201706.677417:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.677419:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.677427:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677440:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478851072 478855167) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677449:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677456:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677462:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.677469:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478851072, 478855168) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.677475:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116907 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.677480:0:17463:0:(rw26.c:898:ll_write_end()) pos 478851072, len 4096, copied 4096 00000080:00200000:0.0:1788201706.677484:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ccde4d00[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116907] 00000080:00200000:0.0:1788201706.677487:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006c100b6d fffffc0001029 2:0 ffff93c0c1385620 116907 lru 00000080:00200000:0.0:1788201706.677493:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001d7d3491 116907: 1< 2 - - > 2< 478851072 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68492 - -> 00000080:00200000:0.0:1788201706.677495:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ccde4d00 00000080:00200000:0.0:1788201706.677496:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.677498:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.677507:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.677525:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.677528:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.677531:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.677541:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.677543:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.677546:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.677550:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677558:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.677567:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.677570:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.677574:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.677578:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116907, 116907] 00000080:00200000:0.0:1788201706.677588:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.677592:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.677621:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.677625:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.677642:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478855168, count: 4096 00000080:00200000:0.0:1788201706.677652:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478855168, bytes: 4096 00000080:00200000:0.0:1788201706.677654:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.677658:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.677660:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478855168, 478859264) 0 0 00000080:00200000:0.0:1788201706.677662:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.677667:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116908, 116908] 00020000:00200000:0.0:1788201706.677670:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478855168 chunk: [478855168, 478859264) 478859264, 4096 00020000:00200000:0.0:1788201706.677672:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.677684:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116908, 116908] 00000008:00010000:0.0:1788201706.677689:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.677692:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.677700:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677708:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478855168 478859263) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677716:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677723:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677729:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.677738:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478855168, 478859264) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.677743:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116908 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.677748:0:17463:0:(rw26.c:898:ll_write_end()) pos 478855168, len 4096, copied 4096 00000080:00200000:0.0:1788201706.677752:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000047882f78[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116908] 00000080:00200000:0.0:1788201706.677755:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f421d198 fffffc0001029 2:0 ffff93c0c1385d20 116908 lru 00000080:00200000:0.0:1788201706.677762:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000df3f4dd2 116908: 1< 2 - - > 2< 478855168 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68493 - -> 00000080:00200000:0.0:1788201706.677764:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000047882f78 00000080:00200000:0.0:1788201706.677765:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.677767:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.677776:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.677785:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.677787:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.677789:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.677796:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.677798:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.677799:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.677802:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677807:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.677812:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.677814:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.677816:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.677818:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116908, 116908] 00000080:00200000:0.0:1788201706.677826:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.677828:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.677863:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.677865:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.677887:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478859264, count: 4096 00000080:00200000:0.0:1788201706.677895:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478859264, bytes: 4096 00000080:00200000:0.0:1788201706.677897:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.677899:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.677900:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478859264, 478863360) 0 0 00000080:00200000:0.0:1788201706.677902:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.677905:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116909, 116909] 00020000:00200000:0.0:1788201706.677907:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478859264 chunk: [478859264, 478863360) 478863360, 4096 00020000:00200000:0.0:1788201706.677909:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.677916:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116909, 116909] 00000008:00010000:0.0:1788201706.677919:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.677921:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.677929:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677938:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478859264 478863359) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677946:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677954:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.677960:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.677968:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478859264, 478863360) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.677973:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116909 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.677978:0:17463:0:(rw26.c:898:ll_write_end()) pos 478859264, len 4096, copied 4096 00000080:00200000:0.0:1788201706.677983:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000869fdc5b[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116909] 00000080:00200000:0.0:1788201706.677986:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000007f92e03 fffffc0001029 2:0 ffff93c0c13859a0 116909 lru 00000080:00200000:0.0:1788201706.677992:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000012a61ef6 116909: 1< 2 - - > 2< 478859264 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68494 - -> 00000080:00200000:0.0:1788201706.677994:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000869fdc5b 00000080:00200000:0.0:1788201706.677996:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.677998:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.678007:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.678036:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.678038:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.678040:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.678048:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.678050:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.678052:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.678054:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678059:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.678065:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.678066:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.678068:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.678070:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116909, 116909] 00000080:00200000:0.0:1788201706.678077:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.678079:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.678098:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.678101:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.678111:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478863360, count: 4096 00000080:00200000:0.0:1788201706.678117:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478863360, bytes: 4096 00000080:00200000:0.0:1788201706.678119:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.678121:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.678123:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478863360, 478867456) 0 0 00000080:00200000:0.0:1788201706.678124:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.678128:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116910, 116910] 00020000:00200000:0.0:1788201706.678130:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478863360 chunk: [478863360, 478867456) 478867456, 4096 00020000:00200000:0.0:1788201706.678132:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.678138:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116910, 116910] 00000008:00010000:0.0:1788201706.678141:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.678143:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.678148:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678154:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478863360 478867455) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678160:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678164:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678168:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.678176:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478863360, 478867456) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.678181:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116910 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.678186:0:17463:0:(rw26.c:898:ll_write_end()) pos 478863360, len 4096, copied 4096 00000080:00200000:0.0:1788201706.678190:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a2e60481[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116910] 00000080:00200000:0.0:1788201706.678193:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009ca9eb3d fffffc0001029 2:0 ffff93c0c1385b60 116910 lru 00000080:00200000:0.0:1788201706.678199:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009fcf41e3 116910: 1< 2 - - > 2< 478863360 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68495 - -> 00000080:00200000:0.0:1788201706.678201:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a2e60481 00000080:00200000:0.0:1788201706.678203:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.678205:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.678213:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.678225:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.678229:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.678231:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.678251:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.678254:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.678257:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.678261:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678268:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.678273:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.678279:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.678282:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.678285:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116910, 116910] 00000080:00200000:0.0:1788201706.678292:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.678294:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.678322:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.678325:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.678342:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478867456, count: 4096 00000080:00200000:0.0:1788201706.678353:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478867456, bytes: 4096 00000080:00200000:0.0:1788201706.678356:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.678360:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.678361:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478867456, 478871552) 0 0 00000080:00200000:0.0:1788201706.678364:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.678370:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116911, 116911] 00020000:00200000:0.0:1788201706.678373:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478867456 chunk: [478867456, 478871552) 478871552, 4096 00020000:00200000:0.0:1788201706.678375:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.678387:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116911, 116911] 00000008:00010000:0.0:1788201706.678391:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.678393:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.678401:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678409:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478867456 478871551) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678418:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678425:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678431:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.678438:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478867456, 478871552) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.678442:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116911 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.678447:0:17463:0:(rw26.c:898:ll_write_end()) pos 478867456, len 4096, copied 4096 00000080:00200000:0.0:1788201706.678450:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f3909c4f[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116911] 00000080:00200000:0.0:1788201706.678453:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005f51b9f3 fffffc0001029 2:0 ffff93c0c1385c40 116911 lru 00000080:00200000:0.0:1788201706.678459:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002d020aeb 116911: 1< 2 - - > 2< 478867456 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68496 - -> 00000080:00200000:0.0:1788201706.678463:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f3909c4f 00000080:00200000:0.0:1788201706.678464:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.678466:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.678474:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.678486:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.678489:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.678491:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.678502:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.678504:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.678507:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.678511:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678519:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.678527:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.678529:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.678534:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.678537:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116911, 116911] 00000080:00200000:0.0:1788201706.678548:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.678551:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.678581:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.678584:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.678601:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478871552, count: 4096 00000080:00200000:0.0:1788201706.678612:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478871552, bytes: 4096 00000080:00200000:0.0:1788201706.678615:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.678618:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.678620:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478871552, 478875648) 0 0 00000080:00200000:0.0:1788201706.678623:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.678628:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116912, 116912] 00020000:00200000:0.0:1788201706.678631:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478871552 chunk: [478871552, 478875648) 478875648, 4096 00020000:00200000:0.0:1788201706.678633:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.678644:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116912, 116912] 00000008:00010000:0.0:1788201706.678650:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.678652:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.678659:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678668:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478871552 478875647) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678677:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678683:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678690:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.678698:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478871552, 478875648) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.678703:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116912 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.678708:0:17463:0:(rw26.c:898:ll_write_end()) pos 478871552, len 4096, copied 4096 00000080:00200000:0.0:1788201706.678713:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d510f25a[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116912] 00000080:00200000:0.0:1788201706.678716:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000994fdeab fffffc0001029 2:0 ffff93c0c12dc0e0 116912 lru 00000080:00200000:0.0:1788201706.678722:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bcb9a22d 116912: 1< 2 - - > 2< 478871552 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68497 - -> 00000080:00200000:0.0:1788201706.678725:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d510f25a 00000080:00200000:0.0:1788201706.678726:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.678728:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.678736:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.678745:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.678747:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.678748:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.678756:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.678757:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.678759:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.678762:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678766:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.678772:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.678773:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.678776:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.678778:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116912, 116912] 00000080:00200000:0.0:1788201706.678785:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.678787:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.678805:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.678808:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.678819:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478875648, count: 4096 00000080:00200000:0.0:1788201706.678825:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478875648, bytes: 4096 00000080:00200000:0.0:1788201706.678827:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.678844:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.678845:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478875648, 478879744) 0 0 00000080:00200000:0.0:1788201706.678847:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.678850:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116913, 116913] 00020000:00200000:0.0:1788201706.678852:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478875648 chunk: [478875648, 478879744) 478879744, 4096 00020000:00200000:0.0:1788201706.678853:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.678861:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116913, 116913] 00000008:00010000:0.0:1788201706.678864:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.678866:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.678873:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678882:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478875648 478879743) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678891:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678898:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678904:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.678912:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478875648, 478879744) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.678917:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116913 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.678923:0:17463:0:(rw26.c:898:ll_write_end()) pos 478875648, len 4096, copied 4096 00000080:00200000:0.0:1788201706.678927:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b8dcfb7d[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116913] 00000080:00200000:0.0:1788201706.678929:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000061c724e9 fffffc0001029 2:0 ffff93c0c12dc700 116913 lru 00000080:00200000:0.0:1788201706.678934:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ea3f2c68 116913: 1< 2 - - > 2< 478875648 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68498 - -> 00000080:00200000:0.0:1788201706.678937:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b8dcfb7d 00000080:00200000:0.0:1788201706.678938:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.678939:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.678950:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.678961:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.678964:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.678966:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.678975:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.678978:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.678980:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.678984:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.678991:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.679000:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.679002:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.679006:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.679009:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116913, 116913] 00000080:00200000:0.0:1788201706.679032:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.679035:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.679061:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.679064:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.679075:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478879744, count: 4096 00000080:00200000:0.0:1788201706.679082:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478879744, bytes: 4096 00000080:00200000:0.0:1788201706.679083:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.679086:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.679087:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478879744, 478883840) 0 0 00000080:00200000:0.0:1788201706.679088:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.679092:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116914, 116914] 00020000:00200000:0.0:1788201706.679094:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478879744 chunk: [478879744, 478883840) 478883840, 4096 00020000:00200000:0.0:1788201706.679095:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.679103:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116914, 116914] 00000008:00010000:0.0:1788201706.679106:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.679107:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.679113:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679118:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478879744 478883839) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679124:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679129:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679133:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.679137:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478879744, 478883840) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.679141:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116914 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.679144:0:17463:0:(rw26.c:898:ll_write_end()) pos 478879744, len 4096, copied 4096 00000080:00200000:0.0:1788201706.679147:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004a4b9ac7[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116914] 00000080:00200000:0.0:1788201706.679149:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fa4a2e7b fffffc0001029 2:0 ffff93c0c12dc000 116914 lru 00000080:00200000:0.0:1788201706.679153:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000039eea7cb 116914: 1< 2 - - > 2< 478879744 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68499 - -> 00000080:00200000:0.0:1788201706.679154:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004a4b9ac7 00000080:00200000:0.0:1788201706.679155:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.679156:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.679162:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.679169:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.679171:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.679172:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.679179:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.679181:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.679182:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.679185:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679189:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.679194:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.679196:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.679198:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.679200:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116914, 116914] 00000080:00200000:0.0:1788201706.679206:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.679208:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.679226:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.679228:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.679239:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478883840, count: 4096 00000080:00200000:0.0:1788201706.679249:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478883840, bytes: 4096 00000080:00200000:0.0:1788201706.679251:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.679255:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.679257:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478883840, 478887936) 0 0 00000080:00200000:0.0:1788201706.679259:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.679264:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116915, 116915] 00020000:00200000:0.0:1788201706.679267:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478883840 chunk: [478883840, 478887936) 478887936, 4096 00020000:00200000:0.0:1788201706.679269:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.679281:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116915, 116915] 00000008:00010000:0.0:1788201706.679286:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.679288:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.679296:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679306:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478883840 478887935) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679314:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679321:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679328:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.679340:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478883840, 478887936) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.679346:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116915 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.679351:0:17463:0:(rw26.c:898:ll_write_end()) pos 478883840, len 4096, copied 4096 00000080:00200000:0.0:1788201706.679355:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004917834b[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116915] 00000080:00200000:0.0:1788201706.679358:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000be46a808 fffffc0001029 2:0 ffff93c0c12dca80 116915 lru 00000080:00200000:0.0:1788201706.679364:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000de804691 116915: 1< 2 - - > 2< 478883840 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68500 - -> 00000080:00200000:0.0:1788201706.679366:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004917834b 00000080:00200000:0.0:1788201706.679367:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.679369:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.679378:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.679390:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.679393:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.679395:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.679404:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.679407:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.679409:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.679413:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679421:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.679429:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.679431:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.679436:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.679439:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116915, 116915] 00000080:00200000:0.0:1788201706.679450:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.679454:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.679483:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.679487:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.679502:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478887936, count: 4096 00000080:00200000:0.0:1788201706.679514:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478887936, bytes: 4096 00000080:00200000:0.0:1788201706.679517:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.679520:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.679522:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478887936, 478892032) 0 0 00000080:00200000:0.0:1788201706.679524:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.679529:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116916, 116916] 00020000:00200000:0.0:1788201706.679532:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478887936 chunk: [478887936, 478892032) 478892032, 4096 00020000:00200000:0.0:1788201706.679534:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.679545:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116916, 116916] 00000008:00010000:0.0:1788201706.679550:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.679552:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.679560:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679568:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478887936 478892031) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679577:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679584:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679595:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.679603:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478887936, 478892032) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.679608:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116916 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.679613:0:17463:0:(rw26.c:898:ll_write_end()) pos 478887936, len 4096, copied 4096 00000080:00200000:0.0:1788201706.679617:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000246b78e[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116916] 00000080:00200000:0.0:1788201706.679620:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d80b318e fffffc0001029 2:0 ffff93c0c12dd9a0 116916 lru 00000080:00200000:0.0:1788201706.679626:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000550e307c 116916: 1< 2 - - > 2< 478887936 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68501 - -> 00000080:00200000:0.0:1788201706.679629:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000246b78e 00000080:00200000:0.0:1788201706.679630:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.679632:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.679640:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.679652:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.679655:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.679657:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.679666:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.679669:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.679672:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.679676:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679684:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.679697:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.679699:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.679704:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.679707:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116916, 116916] 00000080:00200000:0.0:1788201706.679719:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.679722:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.679751:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.679755:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.679772:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478892032, count: 4096 00000080:00200000:0.0:1788201706.679783:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478892032, bytes: 4096 00000080:00200000:0.0:1788201706.679786:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.679790:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.679791:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478892032, 478896128) 0 0 00000080:00200000:0.0:1788201706.679794:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.679799:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116917, 116917] 00020000:00200000:0.0:1788201706.679802:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478892032 chunk: [478892032, 478896128) 478896128, 4096 00020000:00200000:0.0:1788201706.679804:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.679816:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116917, 116917] 00000008:00010000:0.0:1788201706.679821:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.679823:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.679848:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679857:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478892032 478896127) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679866:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679878:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679885:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.679893:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478892032, 478896128) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.679898:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116917 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.679904:0:17463:0:(rw26.c:898:ll_write_end()) pos 478892032, len 4096, copied 4096 00000080:00200000:0.0:1788201706.679908:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009cd1ee41[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116917] 00000080:00200000:0.0:1788201706.679911:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e346e9ad fffffc0001029 2:0 ffff93c0c12ddb60 116917 lru 00000080:00200000:0.0:1788201706.679917:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001d3e6074 116917: 1< 2 - - > 2< 478892032 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68502 - -> 00000080:00200000:0.0:1788201706.679919:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009cd1ee41 00000080:00200000:0.0:1788201706.679920:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.679922:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.679931:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.679943:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.679946:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.679948:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.679957:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.679960:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.679962:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.679966:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.679979:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.679985:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.679986:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.679989:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.679991:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116917, 116917] 00000080:00200000:0.0:1788201706.679999:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.680001:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.680037:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.680040:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.680051:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478896128, count: 4096 00000080:00200000:0.0:1788201706.680058:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478896128, bytes: 4096 00000080:00200000:0.0:1788201706.680060:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.680062:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.680063:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478896128, 478900224) 0 0 00000080:00200000:0.0:1788201706.680065:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.680068:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116918, 116918] 00020000:00200000:0.0:1788201706.680070:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478896128 chunk: [478896128, 478900224) 478900224, 4096 00020000:00200000:0.0:1788201706.680072:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.680078:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116918, 116918] 00000008:00010000:0.0:1788201706.680082:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.680083:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.680088:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680095:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478896128 478900223) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680104:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680109:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680113:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.680117:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478896128, 478900224) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.680120:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116918 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.680123:0:17463:0:(rw26.c:898:ll_write_end()) pos 478896128, len 4096, copied 4096 00000080:00200000:0.0:1788201706.680126:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006e918d6c[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116918] 00000080:00200000:0.0:1788201706.680127:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f991d520 fffffc0001029 2:0 ffff93c0c12ddc40 116918 lru 00000080:00200000:0.0:1788201706.680131:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000034646dc8 116918: 1< 2 - - > 2< 478896128 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68503 - -> 00000080:00200000:0.0:1788201706.680132:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006e918d6c 00000080:00200000:0.0:1788201706.680133:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.680134:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.680140:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.680148:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.680149:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.680151:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.680157:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.680159:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.680161:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.680164:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680168:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.680174:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.680175:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.680177:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.680179:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116918, 116918] 00000080:00200000:0.0:1788201706.680186:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.680188:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.680205:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.680208:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.680218:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478900224, count: 4096 00000080:00200000:0.0:1788201706.680224:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478900224, bytes: 4096 00000080:00200000:0.0:1788201706.680225:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.680227:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.680229:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478900224, 478904320) 0 0 00000080:00200000:0.0:1788201706.680230:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.680233:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116919, 116919] 00020000:00200000:0.0:1788201706.680235:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478900224 chunk: [478900224, 478904320) 478904320, 4096 00020000:00200000:0.0:1788201706.680236:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.680243:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116919, 116919] 00000008:00010000:0.0:1788201706.680246:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.680247:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.680252:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680261:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478900224 478904319) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680267:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680275:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680282:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.680289:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478900224, 478904320) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.680294:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116919 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.680299:0:17463:0:(rw26.c:898:ll_write_end()) pos 478900224, len 4096, copied 4096 00000080:00200000:0.0:1788201706.680303:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a0f7cf10[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116919] 00000080:00200000:0.0:1788201706.680305:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e9963b40 fffffc0001029 2:0 ffff93c0c12dd540 116919 lru 00000080:00200000:0.0:1788201706.680311:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b4cf050d 116919: 1< 2 - - > 2< 478900224 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68504 - -> 00000080:00200000:0.0:1788201706.680313:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a0f7cf10 00000080:00200000:0.0:1788201706.680314:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.680316:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.680324:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.680335:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.680338:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.680341:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.680349:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.680352:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.680354:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.680358:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680366:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.680374:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.680376:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.680380:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.680383:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116919, 116919] 00000080:00200000:0.0:1788201706.680393:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.680396:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.680423:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.680426:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.680437:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478904320, count: 4096 00000080:00200000:0.0:1788201706.680443:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478904320, bytes: 4096 00000080:00200000:0.0:1788201706.680445:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.680447:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.680448:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478904320, 478908416) 0 0 00000080:00200000:0.0:1788201706.680450:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.680453:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116920, 116920] 00020000:00200000:0.0:1788201706.680455:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478904320 chunk: [478904320, 478908416) 478908416, 4096 00020000:00200000:0.0:1788201706.680457:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.680464:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116920, 116920] 00000008:00010000:0.0:1788201706.680467:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.680468:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.680474:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680480:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478904320 478908415) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680486:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680490:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680494:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.680499:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478904320, 478908416) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.680502:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116920 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.680506:0:17463:0:(rw26.c:898:ll_write_end()) pos 478904320, len 4096, copied 4096 00000080:00200000:0.0:1788201706.680508:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000791d3645[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116920] 00000080:00200000:0.0:1788201706.680510:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003cd83d62 fffffc0001029 2:0 ffff93c0c12dd460 116920 lru 00000080:00200000:0.0:1788201706.680514:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000080b3da2 116920: 1< 2 - - > 2< 478904320 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68505 - -> 00000080:00200000:0.0:1788201706.680515:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000791d3645 00000080:00200000:0.0:1788201706.680516:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.680517:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.680523:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.680532:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.680535:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.680537:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.680546:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.680549:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.680551:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.680555:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680563:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.680571:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.680573:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.680577:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.680580:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116920, 116920] 00000080:00200000:0.0:1788201706.680591:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.680594:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1788201706.680621:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:0.0:1788201706.680625:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1788201706.680642:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 478908416, count: 4096 00000080:00200000:0.0:1788201706.680653:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 478908416, bytes: 4096 00000080:00200000:0.0:1788201706.680655:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1788201706.680659:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:0.0:1788201706.680661:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [478908416, 478912512) 0 0 00000080:00200000:0.0:1788201706.680663:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1788201706.680668:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [116921, 116921] 00020000:00200000:0.0:1788201706.680670:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 478908416 chunk: [478908416, 478912512) 478912512, 4096 00020000:00200000:0.0:1788201706.680673:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201706.680684:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [116921, 116921] 00000008:00010000:0.0:1788201706.680690:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1788201706.680692:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1c539acb0, osc lock ffff93c1c82651d0, flags 0x0 00010000:00010000:0.0:1788201706.680700:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680709:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (478908416 478912511) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 279/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680717:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680725:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 283/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680732:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 282/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.680739:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [478908416, 478912512) append 0 append lockpos 0 00000080:00200000:0.0:1788201706.680744:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 116921 of 0 to 4096 bytes 00000080:00200000:0.0:1788201706.680749:0:17463:0:(rw26.c:898:ll_write_end()) pos 478908416, len 4096, copied 4096 00000080:00200000:0.0:1788201706.680754:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a30a8fcd[3 0000000075f4bf38 2 1 00000000232076bf 0 0 116921] 00000080:00200000:0.0:1788201706.680757:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000d911977 fffffc0001029 2:0 ffff93c0c12dd0e0 116921 lru 00000080:00200000:0.0:1788201706.680762:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005c53ab63 116921: 1< 2 - - > 2< 478908416 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 15335424 - | + - + - > 5< + - + - | 0 - | 68506 - -> 00000080:00200000:0.0:1788201706.680764:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a30a8fcd 00000080:00200000:0.0:1788201706.680766:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1788201706.680767:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1788201706.680776:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 00000000bd204a7f ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1788201706.680792:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:0.0:1788201706.680795:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1788201706.680797:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1788201706.680807:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_release():245 00000020:00010000:0.0:1788201706.680809:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1788201706.680812:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c539acb0 (ffff93c1c85e7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1788201706.680816:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 281/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1788201706.680823:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 280/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1788201706.680855:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2e02038 (ffff93c1c85e7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1788201706.680858:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c539acb0 (ffff93c1c85e7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1788201706.680863:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1788201706.680866:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [116921, 116921] 00000080:00200000:0.0:1788201706.680882:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201706.680884:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00010000:00010000:3.0:1788201706.970817:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.970825:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.970843:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484536320, 484540416) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.970847:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118295 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.970850:0:17463:0:(rw26.c:898:ll_write_end()) pos 484536320, len 4096, copied 4096 00000080:00200000:3.0:1788201706.970853:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000082215a07[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118295] 00000080:00200000:3.0:1788201706.970855:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000079bb701a fffffc0001029 2:0 ffff93c0c122e7e0 118295 lru 00000080:00200000:3.0:1788201706.970858:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003566ceaa 118295: 1< 2 - - > 2< 484536320 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69880 - -> 00000080:00200000:3.0:1788201706.970860:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000082215a07 00000080:00200000:3.0:1788201706.970861:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.970862:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.970868:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.970875:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.970877:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.970879:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.970885:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.970886:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.970888:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.970890:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.970895:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.970904:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.970906:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.970908:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.970910:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118295, 118295] 00000080:00200000:3.0:1788201706.970917:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.970919:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.970937:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.970939:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.970950:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484540416, count: 4096 00000080:00200000:3.0:1788201706.970956:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484540416, bytes: 4096 00000080:00200000:3.0:1788201706.970957:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.970960:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.970961:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484540416, 484544512) 0 0 00000080:00200000:3.0:1788201706.970962:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.970965:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118296, 118296] 00020000:00200000:3.0:1788201706.970967:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484540416 chunk: [484540416, 484544512) 484544512, 4096 00020000:00200000:3.0:1788201706.970969:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.970976:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118296, 118296] 00000008:00010000:3.0:1788201706.970979:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.970980:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.970985:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.970991:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484540416 484544511) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.970997:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971005:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971024:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.971030:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484540416, 484544512) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.971033:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118296 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.971037:0:17463:0:(rw26.c:898:ll_write_end()) pos 484540416, len 4096, copied 4096 00000080:00200000:3.0:1788201706.971040:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004d6f77fb[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118296] 00000080:00200000:3.0:1788201706.971041:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a3e34aa0 fffffc0001029 2:0 ffff93c0c122e2a0 118296 lru 00000080:00200000:3.0:1788201706.971045:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cd69ebdf 118296: 1< 2 - - > 2< 484540416 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69881 - -> 00000080:00200000:3.0:1788201706.971047:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004d6f77fb 00000080:00200000:3.0:1788201706.971047:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.971049:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.971054:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.971062:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.971064:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.971065:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.971072:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.971074:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.971075:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.971078:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971086:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.971091:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.971093:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.971095:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.971097:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118296, 118296] 00000080:00200000:3.0:1788201706.971104:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.971106:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.971125:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.971127:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.971137:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484544512, count: 4096 00000080:00200000:3.0:1788201706.971143:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484544512, bytes: 4096 00000080:00200000:3.0:1788201706.971145:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.971147:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.971148:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484544512, 484548608) 0 0 00000080:00200000:3.0:1788201706.971150:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.971153:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118297, 118297] 00020000:00200000:3.0:1788201706.971155:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484544512 chunk: [484544512, 484548608) 484548608, 4096 00020000:00200000:3.0:1788201706.971157:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.971164:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118297, 118297] 00000008:00010000:3.0:1788201706.971167:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.971168:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.971173:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971179:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484544512 484548607) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971188:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971193:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971196:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.971201:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484544512, 484548608) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.971204:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118297 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.971207:0:17463:0:(rw26.c:898:ll_write_end()) pos 484544512, len 4096, copied 4096 00000080:00200000:3.0:1788201706.971210:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000e097106[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118297] 00000080:00200000:3.0:1788201706.971212:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001fa84966 fffffc0001029 2:0 ffff93c0c122ee00 118297 lru 00000080:00200000:3.0:1788201706.971216:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000bf70847 118297: 1< 2 - - > 2< 484544512 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69882 - -> 00000080:00200000:3.0:1788201706.971217:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000e097106 00000080:00200000:3.0:1788201706.971218:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.971219:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.971225:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.971232:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.971234:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.971235:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.971241:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.971243:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.971245:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.971247:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971252:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.971256:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.971258:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.971260:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.971262:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118297, 118297] 00000080:00200000:3.0:1788201706.971269:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.971271:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.971289:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.971291:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.971301:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484548608, count: 4096 00000080:00200000:3.0:1788201706.971307:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484548608, bytes: 4096 00000080:00200000:3.0:1788201706.971308:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.971311:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.971312:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484548608, 484552704) 0 0 00000080:00200000:3.0:1788201706.971313:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.971317:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118298, 118298] 00020000:00200000:3.0:1788201706.971318:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484548608 chunk: [484548608, 484552704) 484552704, 4096 00020000:00200000:3.0:1788201706.971320:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.971329:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118298, 118298] 00000008:00010000:3.0:1788201706.971334:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.971336:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.971343:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971358:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484548608 484552703) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971367:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971375:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971379:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.971385:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484548608, 484552704) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.971388:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118298 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.971392:0:17463:0:(rw26.c:898:ll_write_end()) pos 484548608, len 4096, copied 4096 00000080:00200000:3.0:1788201706.971394:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a1375309[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118298] 00000080:00200000:3.0:1788201706.971396:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a2d9f78c fffffc0001029 2:0 ffff93c0c122e1c0 118298 lru 00000080:00200000:3.0:1788201706.971400:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006e98aab6 118298: 1< 2 - - > 2< 484548608 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69883 - -> 00000080:00200000:3.0:1788201706.971401:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a1375309 00000080:00200000:3.0:1788201706.971402:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.971403:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.971409:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.971417:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.971418:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.971420:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.971426:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.971428:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.971429:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.971432:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971437:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.971442:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.971443:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.971445:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.971447:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118298, 118298] 00000080:00200000:3.0:1788201706.971458:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.971461:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.971489:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.971493:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.971508:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484552704, count: 4096 00000080:00200000:3.0:1788201706.971518:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484552704, bytes: 4096 00000080:00200000:3.0:1788201706.971520:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.971524:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.971525:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484552704, 484556800) 0 0 00000080:00200000:3.0:1788201706.971528:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.971533:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118299, 118299] 00020000:00200000:3.0:1788201706.971535:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484552704 chunk: [484552704, 484556800) 484556800, 4096 00020000:00200000:3.0:1788201706.971537:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.971549:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118299, 118299] 00000008:00010000:3.0:1788201706.971554:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.971556:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.971564:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971572:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484552704 484556799) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971581:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971588:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971595:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.971603:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484552704, 484556800) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.971607:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118299 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.971613:0:17463:0:(rw26.c:898:ll_write_end()) pos 484552704, len 4096, copied 4096 00000080:00200000:3.0:1788201706.971617:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000087228508[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118299] 00000080:00200000:3.0:1788201706.971620:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003eb739f7 fffffc0001029 2:0 ffff93c0c122f700 118299 lru 00000080:00200000:3.0:1788201706.971625:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008eba4c48 118299: 1< 2 - - > 2< 484552704 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69884 - -> 00000080:00200000:3.0:1788201706.971627:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000087228508 00000080:00200000:3.0:1788201706.971628:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.971630:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.971639:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.971650:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.971653:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.971655:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.971664:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.971667:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.971669:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.971672:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971679:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.971687:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.971689:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.971693:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.971696:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118299, 118299] 00000080:00200000:3.0:1788201706.971707:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.971711:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.971739:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.971743:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.971759:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484556800, count: 4096 00000080:00200000:3.0:1788201706.971770:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484556800, bytes: 4096 00000080:00200000:3.0:1788201706.971772:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.971776:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.971778:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484556800, 484560896) 0 0 00000080:00200000:3.0:1788201706.971780:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.971786:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118300, 118300] 00020000:00200000:3.0:1788201706.971789:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484556800 chunk: [484556800, 484560896) 484560896, 4096 00020000:00200000:3.0:1788201706.971791:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.971803:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118300, 118300] 00000008:00010000:3.0:1788201706.971808:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.971810:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.971817:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971826:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484556800 484560895) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971852:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971860:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971868:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.971875:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484556800, 484560896) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.971880:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118300 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.971886:0:17463:0:(rw26.c:898:ll_write_end()) pos 484556800, len 4096, copied 4096 00000080:00200000:3.0:1788201706.971889:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003e6e5b47[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118300] 00000080:00200000:3.0:1788201706.971894:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002fe8e3b1 fffffc0001029 2:0 ffff93c0c122f000 118300 lru 00000080:00200000:3.0:1788201706.971900:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000049dfe467 118300: 1< 2 - - > 2< 484556800 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69885 - -> 00000080:00200000:3.0:1788201706.971902:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003e6e5b47 00000080:00200000:3.0:1788201706.971904:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.971906:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.971915:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.971932:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.971935:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.971937:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.971946:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.971948:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.971951:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.971956:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.971963:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.971971:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.971973:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.971978:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.971981:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118300, 118300] 00000080:00200000:3.0:1788201706.971994:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.971997:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.972040:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.972044:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.972061:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484560896, count: 4096 00000080:00200000:3.0:1788201706.972070:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484560896, bytes: 4096 00000080:00200000:3.0:1788201706.972073:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.972076:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.972078:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484560896, 484564992) 0 0 00000080:00200000:3.0:1788201706.972080:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.972085:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118301, 118301] 00020000:00200000:3.0:1788201706.972088:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484560896 chunk: [484560896, 484564992) 484564992, 4096 00020000:00200000:3.0:1788201706.972090:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.972101:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118301, 118301] 00000008:00010000:3.0:1788201706.972105:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.972107:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.972115:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972122:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484560896 484564991) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972131:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972139:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972146:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.972155:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484560896, 484564992) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.972161:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118301 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.972166:0:17463:0:(rw26.c:898:ll_write_end()) pos 484560896, len 4096, copied 4096 00000080:00200000:3.0:1788201706.972169:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004dece0dc[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118301] 00000080:00200000:3.0:1788201706.972173:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000df2fe10 fffffc0001029 2:0 ffff93c0c122fa80 118301 lru 00000080:00200000:3.0:1788201706.972178:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002f5a8271 118301: 1< 2 - - > 2< 484560896 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69886 - -> 00000080:00200000:3.0:1788201706.972180:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004dece0dc 00000080:00200000:3.0:1788201706.972181:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.972184:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.972192:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.972203:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.972206:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.972208:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.972217:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.972220:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.972222:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.972227:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972235:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.972242:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.972245:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.972249:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.972253:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118301, 118301] 00000080:00200000:3.0:1788201706.972264:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.972268:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.972298:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.972302:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.972319:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484564992, count: 4096 00000080:00200000:3.0:1788201706.972329:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484564992, bytes: 4096 00000080:00200000:3.0:1788201706.972332:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.972336:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.972337:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484564992, 484569088) 0 0 00000080:00200000:3.0:1788201706.972340:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.972344:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118302, 118302] 00020000:00200000:3.0:1788201706.972347:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484564992 chunk: [484564992, 484569088) 484569088, 4096 00020000:00200000:3.0:1788201706.972350:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.972363:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118302, 118302] 00000008:00010000:3.0:1788201706.972369:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.972371:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.972378:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972387:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484564992 484569087) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972396:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972403:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972410:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.972418:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484564992, 484569088) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.972422:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118302 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.972428:0:17463:0:(rw26.c:898:ll_write_end()) pos 484564992, len 4096, copied 4096 00000080:00200000:3.0:1788201706.972431:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e1a908bd[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118302] 00000080:00200000:3.0:1788201706.972434:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000027219b08 fffffc0001029 2:0 ffff93c0c122f380 118302 lru 00000080:00200000:3.0:1788201706.972440:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000114b3172 118302: 1< 2 - - > 2< 484564992 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69887 - -> 00000080:00200000:3.0:1788201706.972444:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e1a908bd 00000080:00200000:3.0:1788201706.972445:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.972446:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.972454:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.972466:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.972469:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.972471:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.972480:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.972483:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.972486:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.972490:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972497:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.972504:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.972507:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.972511:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.972514:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118302, 118302] 00000080:00200000:3.0:1788201706.972525:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.972528:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.972557:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.972560:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.972577:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484569088, count: 4096 00000080:00200000:3.0:1788201706.972588:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484569088, bytes: 4096 00000080:00200000:3.0:1788201706.972591:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.972595:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.972596:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484569088, 484573184) 0 0 00000080:00200000:3.0:1788201706.972599:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.972604:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118303, 118303] 00020000:00200000:3.0:1788201706.972607:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484569088 chunk: [484569088, 484573184) 484573184, 4096 00020000:00200000:3.0:1788201706.972609:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.972621:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118303, 118303] 00000008:00010000:3.0:1788201706.972626:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.972629:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.972636:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972645:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484569088 484573183) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972653:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972661:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972667:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.972674:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484569088, 484573184) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.972678:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118303 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.972684:0:17463:0:(rw26.c:898:ll_write_end()) pos 484569088, len 4096, copied 4096 00000080:00200000:3.0:1788201706.972687:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006fe35903[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118303] 00000080:00200000:3.0:1788201706.972691:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000577b0d1f fffffc0001029 2:0 ffff93c0c122f8c0 118303 lru 00000080:00200000:3.0:1788201706.972697:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000db3d6c8b 118303: 1< 2 - - > 2< 484569088 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69888 - -> 00000080:00200000:3.0:1788201706.972699:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006fe35903 00000080:00200000:3.0:1788201706.972700:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.972702:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.972710:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.972722:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.972724:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.972727:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.972735:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.972738:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.972741:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.972745:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972753:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.972774:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.972777:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.972781:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.972784:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118303, 118303] 00000080:00200000:3.0:1788201706.972796:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.972799:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.972826:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.972846:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.972864:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484573184, count: 4096 00000080:00200000:3.0:1788201706.972875:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484573184, bytes: 4096 00000080:00200000:3.0:1788201706.972878:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.972881:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.972882:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484573184, 484577280) 0 0 00000080:00200000:3.0:1788201706.972885:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.972890:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118304, 118304] 00020000:00200000:3.0:1788201706.972893:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484573184 chunk: [484573184, 484577280) 484577280, 4096 00020000:00200000:3.0:1788201706.972895:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.972908:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118304, 118304] 00000008:00010000:3.0:1788201706.972914:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.972916:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.972924:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972932:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484573184 484577279) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972941:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972949:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.972956:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.972964:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484573184, 484577280) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.972969:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118304 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.972974:0:17463:0:(rw26.c:898:ll_write_end()) pos 484573184, len 4096, copied 4096 00000080:00200000:3.0:1788201706.972978:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d8e9029f[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118304] 00000080:00200000:3.0:1788201706.972980:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b0c1143a fffffc0001029 2:0 ffff93c0c122fee0 118304 lru 00000080:00200000:3.0:1788201706.972986:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d10c6393 118304: 1< 2 - - > 2< 484573184 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69889 - -> 00000080:00200000:3.0:1788201706.972989:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d8e9029f 00000080:00200000:3.0:1788201706.972990:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.972992:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.973000:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.973026:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.973029:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.973031:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.973041:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.973044:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.973046:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.973051:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973058:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.973067:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.973069:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.973074:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.973077:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118304, 118304] 00000080:00200000:3.0:1788201706.973089:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.973092:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.973121:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.973125:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.973140:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484577280, count: 4096 00000080:00200000:3.0:1788201706.973150:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484577280, bytes: 4096 00000080:00200000:3.0:1788201706.973153:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.973157:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.973159:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484577280, 484581376) 0 0 00000080:00200000:3.0:1788201706.973161:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.973166:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118305, 118305] 00020000:00200000:3.0:1788201706.973169:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484577280 chunk: [484577280, 484581376) 484581376, 4096 00020000:00200000:3.0:1788201706.973172:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.973184:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118305, 118305] 00000008:00010000:3.0:1788201706.973190:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.973192:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.973199:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973208:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484577280 484581375) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973217:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973224:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973230:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.973237:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484577280, 484581376) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.973243:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118305 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.973248:0:17463:0:(rw26.c:898:ll_write_end()) pos 484577280, len 4096, copied 4096 00000080:00200000:3.0:1788201706.973252:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003b9d55c8[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118305] 00000080:00200000:3.0:1788201706.973255:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ca49a9d0 fffffc0001029 2:0 ffff93c0c122f7e0 118305 lru 00000080:00200000:3.0:1788201706.973260:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002adcc278 118305: 1< 2 - - > 2< 484577280 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69890 - -> 00000080:00200000:3.0:1788201706.973262:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003b9d55c8 00000080:00200000:3.0:1788201706.973263:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.973267:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.973276:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.973287:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.973290:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.973292:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.973300:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.973303:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.973305:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.973309:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973317:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.973325:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.973327:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.973331:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.973335:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118305, 118305] 00000080:00200000:3.0:1788201706.973346:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.973349:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.973378:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.973381:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.973397:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484581376, count: 4096 00000080:00200000:3.0:1788201706.973408:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484581376, bytes: 4096 00000080:00200000:3.0:1788201706.973410:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.973414:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.973415:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484581376, 484585472) 0 0 00000080:00200000:3.0:1788201706.973418:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.973423:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118306, 118306] 00020000:00200000:3.0:1788201706.973426:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484581376 chunk: [484581376, 484585472) 484585472, 4096 00020000:00200000:3.0:1788201706.973428:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.973440:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118306, 118306] 00000008:00010000:3.0:1788201706.973445:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.973447:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.973455:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973463:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484581376 484585471) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973472:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973480:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973486:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.973499:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484581376, 484585472) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.973504:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118306 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.973510:0:17463:0:(rw26.c:898:ll_write_end()) pos 484581376, len 4096, copied 4096 00000080:00200000:3.0:1788201706.973514:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000081f32c88[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118306] 00000080:00200000:3.0:1788201706.973516:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009d6ecec2 fffffc0001029 2:0 ffff93c0c122f2a0 118306 lru 00000080:00200000:3.0:1788201706.973522:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007d560b90 118306: 1< 2 - - > 2< 484581376 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69891 - -> 00000080:00200000:3.0:1788201706.973524:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000081f32c88 00000080:00200000:3.0:1788201706.973525:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.973528:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.973536:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.973547:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.973550:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.973552:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.973561:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.973564:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.973567:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.973571:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973579:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.973587:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.973590:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.973595:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.973598:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118306, 118306] 00000080:00200000:3.0:1788201706.973609:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.973612:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.973640:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.973643:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.973659:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484585472, count: 4096 00000080:00200000:3.0:1788201706.973670:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484585472, bytes: 4096 00000080:00200000:3.0:1788201706.973673:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.973676:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.973678:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484585472, 484589568) 0 0 00000080:00200000:3.0:1788201706.973680:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.973685:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118307, 118307] 00020000:00200000:3.0:1788201706.973687:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484585472 chunk: [484585472, 484589568) 484589568, 4096 00020000:00200000:3.0:1788201706.973689:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.973701:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118307, 118307] 00000008:00010000:3.0:1788201706.973706:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.973708:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.973715:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973724:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484585472 484589567) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973733:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973740:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973753:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.973760:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484585472, 484589568) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.973765:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118307 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.973770:0:17463:0:(rw26.c:898:ll_write_end()) pos 484585472, len 4096, copied 4096 00000080:00200000:3.0:1788201706.973774:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000080d6d035[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118307] 00000080:00200000:3.0:1788201706.973777:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000237362f fffffc0001029 2:0 ffff93c0c122fe00 118307 lru 00000080:00200000:3.0:1788201706.973783:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e7651b10 118307: 1< 2 - - > 2< 484585472 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69892 - -> 00000080:00200000:3.0:1788201706.973785:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000080d6d035 00000080:00200000:3.0:1788201706.973786:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.973788:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.973796:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.973807:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.973810:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.973812:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.973820:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.973823:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.973826:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.973856:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.973864:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.973877:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.973880:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.973884:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.973888:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118307, 118307] 00000080:00200000:3.0:1788201706.973900:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.973903:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.973931:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.973934:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.973950:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484589568, count: 4096 00000080:00200000:3.0:1788201706.973960:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484589568, bytes: 4096 00000080:00200000:3.0:1788201706.973963:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.973967:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.973968:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484589568, 484593664) 0 0 00000080:00200000:3.0:1788201706.973970:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.973975:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118308, 118308] 00020000:00200000:3.0:1788201706.973978:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484589568 chunk: [484589568, 484593664) 484593664, 4096 00020000:00200000:3.0:1788201706.973980:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.973990:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118308, 118308] 00000008:00010000:3.0:1788201706.973996:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.973998:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.974005:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974029:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484589568 484593663) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974038:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974051:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974058:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.974065:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484589568, 484593664) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.974070:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118308 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.974075:0:17463:0:(rw26.c:898:ll_write_end()) pos 484589568, len 4096, copied 4096 00000080:00200000:3.0:1788201706.974079:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f2f15da2[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118308] 00000080:00200000:3.0:1788201706.974082:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000339f6d11 fffffc0001029 2:0 ffff93c0c122f1c0 118308 lru 00000080:00200000:3.0:1788201706.974087:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000824ff283 118308: 1< 2 - - > 2< 484589568 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69893 - -> 00000080:00200000:3.0:1788201706.974089:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f2f15da2 00000080:00200000:3.0:1788201706.974090:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.974092:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.974100:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.974112:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.974115:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.974117:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.974126:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.974128:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.974131:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.974135:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974148:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.974156:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.974158:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.974162:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.974165:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118308, 118308] 00000080:00200000:3.0:1788201706.974177:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.974179:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.974208:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.974212:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.974228:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484593664, count: 4096 00000080:00200000:3.0:1788201706.974239:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484593664, bytes: 4096 00000080:00200000:3.0:1788201706.974241:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.974245:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.974246:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484593664, 484597760) 0 0 00000080:00200000:3.0:1788201706.974249:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.974254:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118309, 118309] 00020000:00200000:3.0:1788201706.974257:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484593664 chunk: [484593664, 484597760) 484597760, 4096 00020000:00200000:3.0:1788201706.974259:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.974272:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118309, 118309] 00000008:00010000:3.0:1788201706.974277:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.974280:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.974287:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974296:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484593664 484597759) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974310:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974319:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974326:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.974333:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484593664, 484597760) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.974338:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118309 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.974344:0:17463:0:(rw26.c:898:ll_write_end()) pos 484593664, len 4096, copied 4096 00000080:00200000:3.0:1788201706.974348:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c93e8972[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118309] 00000080:00200000:3.0:1788201706.974351:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005b77c1a9 fffffc0001029 2:0 ffff93c0c122f620 118309 lru 00000080:00200000:3.0:1788201706.974357:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009da08df6 118309: 1< 2 - - > 2< 484593664 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69894 - -> 00000080:00200000:3.0:1788201706.974359:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c93e8972 00000080:00200000:3.0:1788201706.974360:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.974362:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.974372:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.974384:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.974386:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.974388:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.974397:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.974400:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.974403:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.974407:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974415:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.974423:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.974426:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.974430:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.974433:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118309, 118309] 00000080:00200000:3.0:1788201706.974444:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.974448:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.974476:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.974480:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.974497:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484597760, count: 4096 00000080:00200000:3.0:1788201706.974508:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484597760, bytes: 4096 00000080:00200000:3.0:1788201706.974511:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.974514:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.974516:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484597760, 484601856) 0 0 00000080:00200000:3.0:1788201706.974518:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.974523:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118310, 118310] 00020000:00200000:3.0:1788201706.974526:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484597760 chunk: [484597760, 484601856) 484601856, 4096 00020000:00200000:3.0:1788201706.974529:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.974540:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118310, 118310] 00000008:00010000:3.0:1788201706.974546:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.974548:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.974555:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974569:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484597760 484601855) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974578:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974585:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974593:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.974600:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484597760, 484601856) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.974606:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118310 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.974611:0:17463:0:(rw26.c:898:ll_write_end()) pos 484597760, len 4096, copied 4096 00000080:00200000:3.0:1788201706.974616:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b03b0d89[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118310] 00000080:00200000:3.0:1788201706.974618:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000974784cd fffffc0001029 2:0 ffff93c0c11ae460 118310 lru 00000080:00200000:3.0:1788201706.974624:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f1d3ba8f 118310: 1< 2 - - > 2< 484597760 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69895 - -> 00000080:00200000:3.0:1788201706.974627:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b03b0d89 00000080:00200000:3.0:1788201706.974628:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.974630:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.974638:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.974651:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.974653:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.974656:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.974665:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.974667:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.974670:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.974674:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974682:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.974691:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.974693:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.974698:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.974701:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118310, 118310] 00000080:00200000:3.0:1788201706.974713:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.974716:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.974746:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.974750:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.974766:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484601856, count: 4096 00000080:00200000:3.0:1788201706.974777:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484601856, bytes: 4096 00000080:00200000:3.0:1788201706.974780:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.974783:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.974785:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484601856, 484605952) 0 0 00000080:00200000:3.0:1788201706.974787:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.974792:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118311, 118311] 00020000:00200000:3.0:1788201706.974795:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484601856 chunk: [484601856, 484605952) 484605952, 4096 00020000:00200000:3.0:1788201706.974798:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.974810:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118311, 118311] 00000008:00010000:3.0:1788201706.974815:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.974817:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.974826:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974853:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484601856 484605951) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974862:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974870:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974877:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.974886:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484601856, 484605952) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.974892:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118311 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.974897:0:17463:0:(rw26.c:898:ll_write_end()) pos 484601856, len 4096, copied 4096 00000080:00200000:3.0:1788201706.974902:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009a517aa1[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118311] 00000080:00200000:3.0:1788201706.974905:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000530668cc fffffc0001029 2:0 ffff93c0c11ae0e0 118311 lru 00000080:00200000:3.0:1788201706.974910:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000dd05104f 118311: 1< 2 - - > 2< 484601856 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69896 - -> 00000080:00200000:3.0:1788201706.974912:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009a517aa1 00000080:00200000:3.0:1788201706.974914:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.974916:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.974924:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.974935:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.974938:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.974941:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.974949:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.974951:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.974954:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.974958:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.974965:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.974973:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.974976:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.974979:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.974983:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118311, 118311] 00000080:00200000:3.0:1788201706.974994:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.974998:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.975042:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.975046:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.975062:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484605952, count: 4096 00000080:00200000:3.0:1788201706.975073:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484605952, bytes: 4096 00000080:00200000:3.0:1788201706.975076:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.975079:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.975081:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484605952, 484610048) 0 0 00000080:00200000:3.0:1788201706.975083:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.975088:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118312, 118312] 00020000:00200000:3.0:1788201706.975091:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484605952 chunk: [484605952, 484610048) 484610048, 4096 00020000:00200000:3.0:1788201706.975093:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.975105:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118312, 118312] 00000008:00010000:3.0:1788201706.975111:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.975113:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.975121:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975129:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484605952 484610047) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975138:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975145:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975153:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.975160:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484605952, 484610048) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.975165:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118312 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.975170:0:17463:0:(rw26.c:898:ll_write_end()) pos 484605952, len 4096, copied 4096 00000080:00200000:3.0:1788201706.975174:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000711f9bef[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118312] 00000080:00200000:3.0:1788201706.975177:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000040036091 fffffc0001029 2:0 ffff93c0c11ae700 118312 lru 00000080:00200000:3.0:1788201706.975183:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000532c7f65 118312: 1< 2 - - > 2< 484605952 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69897 - -> 00000080:00200000:3.0:1788201706.975185:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000711f9bef 00000080:00200000:3.0:1788201706.975186:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.975188:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.975197:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.975213:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.975216:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.975219:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.975228:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.975231:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.975234:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.975238:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975245:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.975253:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.975255:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.975260:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.975263:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118312, 118312] 00000080:00200000:3.0:1788201706.975275:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.975278:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.975307:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.975311:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.975328:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484610048, count: 4096 00000080:00200000:3.0:1788201706.975338:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484610048, bytes: 4096 00000080:00200000:3.0:1788201706.975341:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.975345:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.975347:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484610048, 484614144) 0 0 00000080:00200000:3.0:1788201706.975349:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.975354:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118313, 118313] 00020000:00200000:3.0:1788201706.975358:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484610048 chunk: [484610048, 484614144) 484614144, 4096 00020000:00200000:3.0:1788201706.975360:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.975373:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118313, 118313] 00000008:00010000:3.0:1788201706.975378:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.975381:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.975388:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975397:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484610048 484614143) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975407:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975415:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975421:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.975429:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484610048, 484614144) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.975435:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118313 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.975440:0:17463:0:(rw26.c:898:ll_write_end()) pos 484610048, len 4096, copied 4096 00000080:00200000:3.0:1788201706.975444:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ca3e118b[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118313] 00000080:00200000:3.0:1788201706.975447:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d2298e6e fffffc0001029 2:0 ffff93c0c11ae000 118313 lru 00000080:00200000:3.0:1788201706.975452:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004bdeb7ed 118313: 1< 2 - - > 2< 484610048 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69898 - -> 00000080:00200000:3.0:1788201706.975455:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ca3e118b 00000080:00200000:3.0:1788201706.975456:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.975458:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.975466:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.975478:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.975481:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.975483:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.975493:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.975495:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.975498:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.975502:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975509:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.975517:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.975519:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.975523:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.975526:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118313, 118313] 00000080:00200000:3.0:1788201706.975537:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.975540:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.975569:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.975572:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.975588:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484614144, count: 4096 00000080:00200000:3.0:1788201706.975599:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484614144, bytes: 4096 00000080:00200000:3.0:1788201706.975602:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.975606:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.975607:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484614144, 484618240) 0 0 00000080:00200000:3.0:1788201706.975610:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.975615:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118314, 118314] 00020000:00200000:3.0:1788201706.975618:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484614144 chunk: [484614144, 484618240) 484618240, 4096 00020000:00200000:3.0:1788201706.975621:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.975634:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118314, 118314] 00000008:00010000:3.0:1788201706.975640:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.975642:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.975649:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975659:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484614144 484618239) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975668:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975675:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975682:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.975690:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484614144, 484618240) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.975695:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118314 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.975700:0:17463:0:(rw26.c:898:ll_write_end()) pos 484614144, len 4096, copied 4096 00000080:00200000:3.0:1788201706.975704:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000893460ae[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118314] 00000080:00200000:3.0:1788201706.975707:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d22f946d fffffc0001029 2:0 ffff93c0c11aea80 118314 lru 00000080:00200000:3.0:1788201706.975712:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cd3b1462 118314: 1< 2 - - > 2< 484614144 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69899 - -> 00000080:00200000:3.0:1788201706.975716:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000893460ae 00000080:00200000:3.0:1788201706.975718:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.975719:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.975728:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.975740:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.975743:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.975745:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.975754:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.975756:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.975759:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.975763:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975770:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.975778:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.975780:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.975784:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.975787:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118314, 118314] 00000080:00200000:3.0:1788201706.975798:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.975801:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.975844:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.975848:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.975866:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484618240, count: 4096 00000080:00200000:3.0:1788201706.975876:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484618240, bytes: 4096 00000080:00200000:3.0:1788201706.975878:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.975882:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.975883:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484618240, 484622336) 0 0 00000080:00200000:3.0:1788201706.975885:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.975889:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118315, 118315] 00020000:00200000:3.0:1788201706.975892:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484618240 chunk: [484618240, 484622336) 484622336, 4096 00020000:00200000:3.0:1788201706.975894:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.975905:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118315, 118315] 00000008:00010000:3.0:1788201706.975910:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.975912:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.975919:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975928:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484618240 484622335) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975936:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975943:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.975949:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.975955:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484618240, 484622336) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.975960:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118315 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.975965:0:17463:0:(rw26.c:898:ll_write_end()) pos 484618240, len 4096, copied 4096 00000080:00200000:3.0:1788201706.975970:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b0acc35d[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118315] 00000080:00200000:3.0:1788201706.975973:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cebfea94 fffffc0001029 2:0 ffff93c0c13ef2a0 118315 lru 00000080:00200000:3.0:1788201706.975978:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000641bd59d 118315: 1< 2 - - > 2< 484618240 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69900 - -> 00000080:00200000:3.0:1788201706.975981:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b0acc35d 00000080:00200000:3.0:1788201706.975982:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.975983:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.975992:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.976002:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.976004:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.976006:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.976026:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.976029:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.976031:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.976034:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976039:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.976044:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.976045:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.976048:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.976050:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118315, 118315] 00000080:00200000:3.0:1788201706.976058:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.976060:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.976079:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.976082:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.976093:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484622336, count: 4096 00000080:00200000:3.0:1788201706.976099:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484622336, bytes: 4096 00000080:00200000:3.0:1788201706.976101:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.976103:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.976104:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484622336, 484626432) 0 0 00000080:00200000:3.0:1788201706.976105:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.976109:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118316, 118316] 00020000:00200000:3.0:1788201706.976111:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484622336 chunk: [484622336, 484626432) 484626432, 4096 00020000:00200000:3.0:1788201706.976112:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.976120:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118316, 118316] 00000008:00010000:3.0:1788201706.976123:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.976125:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.976130:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976136:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484622336 484626431) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976141:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976146:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976150:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.976156:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484622336, 484626432) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.976161:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118316 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.976166:0:17463:0:(rw26.c:898:ll_write_end()) pos 484622336, len 4096, copied 4096 00000080:00200000:3.0:1788201706.976170:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a972cd43[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118316] 00000080:00200000:3.0:1788201706.976173:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005ffd1c83 fffffc0001029 2:0 ffff93c0c13efe00 118316 lru 00000080:00200000:3.0:1788201706.976179:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f528ac00 118316: 1< 2 - - > 2< 484622336 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69901 - -> 00000080:00200000:3.0:1788201706.976181:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a972cd43 00000080:00200000:3.0:1788201706.976183:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.976184:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.976195:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.976206:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.976209:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.976211:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.976220:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.976223:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.976225:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.976229:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976236:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.976244:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.976247:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.976250:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.976253:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118316, 118316] 00000080:00200000:3.0:1788201706.976264:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.976267:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.976297:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.976301:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.976316:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484626432, count: 4096 00000080:00200000:3.0:1788201706.976328:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484626432, bytes: 4096 00000080:00200000:3.0:1788201706.976330:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.976334:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.976335:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484626432, 484630528) 0 0 00000080:00200000:3.0:1788201706.976338:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.976342:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118317, 118317] 00020000:00200000:3.0:1788201706.976345:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484626432 chunk: [484626432, 484630528) 484630528, 4096 00020000:00200000:3.0:1788201706.976348:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.976360:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118317, 118317] 00000008:00010000:3.0:1788201706.976366:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.976368:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.976376:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976385:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484626432 484630527) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976394:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976402:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976410:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.976416:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484626432, 484630528) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.976420:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118317 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.976424:0:17463:0:(rw26.c:898:ll_write_end()) pos 484626432, len 4096, copied 4096 00000080:00200000:3.0:1788201706.976427:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003d7b9862[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118317] 00000080:00200000:3.0:1788201706.976429:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001f42cbed fffffc0001029 2:0 ffff93c0c13ef1c0 118317 lru 00000080:00200000:3.0:1788201706.976433:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000034e154cc 118317: 1< 2 - - > 2< 484626432 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69902 - -> 00000080:00200000:3.0:1788201706.976434:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003d7b9862 00000080:00200000:3.0:1788201706.976435:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.976436:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.976443:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.976456:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.976460:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.976462:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.976471:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.976474:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.976476:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.976480:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976488:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.976496:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.976498:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.976503:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.976506:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118317, 118317] 00000080:00200000:3.0:1788201706.976517:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.976520:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.976549:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.976552:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.976568:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484630528, count: 4096 00000080:00200000:3.0:1788201706.976578:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484630528, bytes: 4096 00000080:00200000:3.0:1788201706.976581:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.976585:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.976587:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484630528, 484634624) 0 0 00000080:00200000:3.0:1788201706.976590:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.976594:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118318, 118318] 00020000:00200000:3.0:1788201706.976597:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484630528 chunk: [484630528, 484634624) 484634624, 4096 00020000:00200000:3.0:1788201706.976600:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.976611:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118318, 118318] 00000008:00010000:3.0:1788201706.976616:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.976618:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.976625:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976631:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484630528 484634623) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976637:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976642:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976646:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.976654:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484630528, 484634624) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.976658:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118318 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.976661:0:17463:0:(rw26.c:898:ll_write_end()) pos 484630528, len 4096, copied 4096 00000080:00200000:3.0:1788201706.976664:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bbb8740c[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118318] 00000080:00200000:3.0:1788201706.976666:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e2a63092 fffffc0001029 2:0 ffff93c0c13f01c0 118318 lru 00000080:00200000:3.0:1788201706.976670:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007d715ea6 118318: 1< 2 - - > 2< 484630528 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69903 - -> 00000080:00200000:3.0:1788201706.976671:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bbb8740c 00000080:00200000:3.0:1788201706.976672:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.976673:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.976679:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.976687:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.976689:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.976690:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.976696:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.976698:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.976699:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.976702:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976707:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.976712:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.976713:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.976715:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.976717:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118318, 118318] 00000080:00200000:3.0:1788201706.976725:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.976727:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.976745:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.976747:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.976757:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484634624, count: 4096 00000080:00200000:3.0:1788201706.976763:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484634624, bytes: 4096 00000080:00200000:3.0:1788201706.976765:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.976767:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.976768:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484634624, 484638720) 0 0 00000080:00200000:3.0:1788201706.976770:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.976773:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118319, 118319] 00020000:00200000:3.0:1788201706.976775:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484634624 chunk: [484634624, 484638720) 484638720, 4096 00020000:00200000:3.0:1788201706.976776:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.976783:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118319, 118319] 00000008:00010000:3.0:1788201706.976786:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.976788:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.976793:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976798:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484634624 484638719) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976803:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976808:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976815:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.976819:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484634624, 484638720) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.976822:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118319 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.976826:0:17463:0:(rw26.c:898:ll_write_end()) pos 484634624, len 4096, copied 4096 00000080:00200000:3.0:1788201706.976843:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000cb38017d[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118319] 00000080:00200000:3.0:1788201706.976845:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b1ec3553 fffffc0001029 2:0 ffff93c0c13f0620 118319 lru 00000080:00200000:3.0:1788201706.976849:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000477f47f1 118319: 1< 2 - - > 2< 484634624 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69904 - -> 00000080:00200000:3.0:1788201706.976851:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000cb38017d 00000080:00200000:3.0:1788201706.976852:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.976854:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.976862:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.976874:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.976877:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.976879:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.976889:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.976891:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.976894:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.976898:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.976906:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.976918:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.976921:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.976924:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.976928:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118319, 118319] 00000080:00200000:3.0:1788201706.976939:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.976942:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.976972:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.976974:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.976986:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484638720, count: 4096 00000080:00200000:3.0:1788201706.976993:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484638720, bytes: 4096 00000080:00200000:3.0:1788201706.976995:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.976997:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.976998:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484638720, 484642816) 0 0 00000080:00200000:3.0:1788201706.977000:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.977003:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118320, 118320] 00020000:00200000:3.0:1788201706.977005:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484638720 chunk: [484638720, 484642816) 484642816, 4096 00020000:00200000:3.0:1788201706.977007:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.977028:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118320, 118320] 00000008:00010000:3.0:1788201706.977032:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.977034:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.977039:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977045:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484638720 484642815) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977051:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977059:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977063:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.977068:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484638720, 484642816) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.977071:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118320 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.977075:0:17463:0:(rw26.c:898:ll_write_end()) pos 484638720, len 4096, copied 4096 00000080:00200000:3.0:1788201706.977077:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003e22ead7[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118320] 00000080:00200000:3.0:1788201706.977079:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000048e0e277 fffffc0001029 2:0 ffff93c0c13f0d20 118320 lru 00000080:00200000:3.0:1788201706.977082:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cb7014bc 118320: 1< 2 - - > 2< 484638720 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69905 - -> 00000080:00200000:3.0:1788201706.977083:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003e22ead7 00000080:00200000:3.0:1788201706.977084:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.977085:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.977091:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.977099:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.977101:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.977102:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.977108:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.977110:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.977112:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.977114:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977122:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.977128:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.977129:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.977131:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.977133:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118320, 118320] 00000080:00200000:3.0:1788201706.977140:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.977142:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.977160:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.977162:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.977172:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484642816, count: 4096 00000080:00200000:3.0:1788201706.977178:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484642816, bytes: 4096 00000080:00200000:3.0:1788201706.977180:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.977182:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.977183:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484642816, 484646912) 0 0 00000080:00200000:3.0:1788201706.977185:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.977188:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118321, 118321] 00020000:00200000:3.0:1788201706.977190:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484642816 chunk: [484642816, 484646912) 484646912, 4096 00020000:00200000:3.0:1788201706.977191:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.977198:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118321, 118321] 00000008:00010000:3.0:1788201706.977201:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.977202:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.977207:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977213:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484642816 484646911) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977222:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977226:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977230:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.977235:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484642816, 484646912) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.977238:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118321 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.977242:0:17463:0:(rw26.c:898:ll_write_end()) pos 484642816, len 4096, copied 4096 00000080:00200000:3.0:1788201706.977244:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e564b342[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118321] 00000080:00200000:3.0:1788201706.977246:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000022d43185 fffffc0001029 2:0 ffff93c0c13f09a0 118321 lru 00000080:00200000:3.0:1788201706.977250:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000081a66cfc 118321: 1< 2 - - > 2< 484642816 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69906 - -> 00000080:00200000:3.0:1788201706.977251:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e564b342 00000080:00200000:3.0:1788201706.977252:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.977253:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.977259:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.977266:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.977268:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.977269:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.977275:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.977277:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.977279:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.977281:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977286:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.977290:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.977292:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.977294:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.977296:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118321, 118321] 00000080:00200000:3.0:1788201706.977303:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.977305:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.977322:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.977325:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.977335:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484646912, count: 4096 00000080:00200000:3.0:1788201706.977341:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484646912, bytes: 4096 00000080:00200000:3.0:1788201706.977343:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.977345:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.977346:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484646912, 484651008) 0 0 00000080:00200000:3.0:1788201706.977348:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.977352:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118322, 118322] 00020000:00200000:3.0:1788201706.977355:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484646912 chunk: [484646912, 484651008) 484651008, 4096 00020000:00200000:3.0:1788201706.977357:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.977368:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118322, 118322] 00000008:00010000:3.0:1788201706.977373:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.977376:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.977383:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977397:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484646912 484651007) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977406:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977411:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977415:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.977420:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484646912, 484651008) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.977423:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118322 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.977427:0:17463:0:(rw26.c:898:ll_write_end()) pos 484646912, len 4096, copied 4096 00000080:00200000:3.0:1788201706.977430:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f1e46f28[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118322] 00000080:00200000:3.0:1788201706.977431:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000037cec272 fffffc0001029 2:0 ffff93c0c13f0b60 118322 lru 00000080:00200000:3.0:1788201706.977435:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f1e1a382 118322: 1< 2 - - > 2< 484646912 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69907 - -> 00000080:00200000:3.0:1788201706.977436:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f1e46f28 00000080:00200000:3.0:1788201706.977437:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.977438:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.977444:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.977452:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.977453:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.977455:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.977461:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.977463:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.977465:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.977467:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977472:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.977477:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.977478:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.977480:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.977482:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118322, 118322] 00000080:00200000:3.0:1788201706.977490:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.977492:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.977510:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.977513:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.977527:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484651008, count: 4096 00000080:00200000:3.0:1788201706.977538:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484651008, bytes: 4096 00000080:00200000:3.0:1788201706.977541:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.977544:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.977546:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484651008, 484655104) 0 0 00000080:00200000:3.0:1788201706.977548:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.977553:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118323, 118323] 00020000:00200000:3.0:1788201706.977556:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484651008 chunk: [484651008, 484655104) 484655104, 4096 00020000:00200000:3.0:1788201706.977558:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.977570:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118323, 118323] 00000008:00010000:3.0:1788201706.977575:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.977577:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.977584:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977591:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484651008 484655103) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977597:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977601:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977605:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.977610:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484651008, 484655104) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.977614:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118323 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.977617:0:17463:0:(rw26.c:898:ll_write_end()) pos 484651008, len 4096, copied 4096 00000080:00200000:3.0:1788201706.977620:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f96cb205[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118323] 00000080:00200000:3.0:1788201706.977622:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004f12bf97 fffffc0001029 2:0 ffff93c0c13f0c40 118323 lru 00000080:00200000:3.0:1788201706.977626:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000007cba340 118323: 1< 2 - - > 2< 484651008 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69908 - -> 00000080:00200000:3.0:1788201706.977627:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f96cb205 00000080:00200000:3.0:1788201706.977628:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.977629:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.977635:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.977643:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.977645:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.977646:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.977652:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.977654:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.977656:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.977658:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977663:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.977668:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.977669:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.977672:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.977673:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118323, 118323] 00000080:00200000:3.0:1788201706.977681:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.977683:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201706.977701:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201706.977703:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201706.977713:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 484655104, count: 4096 00000080:00200000:3.0:1788201706.977720:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 484655104, bytes: 4096 00000080:00200000:3.0:1788201706.977721:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201706.977724:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201706.977725:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [484655104, 484659200) 0 0 00000080:00200000:3.0:1788201706.977726:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201706.977729:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118324, 118324] 00020000:00200000:3.0:1788201706.977731:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 484655104 chunk: [484655104, 484659200) 484659200, 4096 00020000:00200000:3.0:1788201706.977733:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201706.977740:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118324, 118324] 00000008:00010000:3.0:1788201706.977743:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201706.977744:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201706.977749:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977755:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (484655104 484659199) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977760:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977764:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977768:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.977772:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [484655104, 484659200) append 0 append lockpos 0 00000080:00200000:3.0:1788201706.977775:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118324 of 0 to 4096 bytes 00000080:00200000:3.0:1788201706.977778:0:17463:0:(rw26.c:898:ll_write_end()) pos 484655104, len 4096, copied 4096 00000080:00200000:3.0:1788201706.977780:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000082a3768a[3 0000000075f4bf38 2 1 0000000006fbcd75 0 0 118324] 00000080:00200000:3.0:1788201706.977782:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000086ea52e5 fffffc0001029 2:0 ffff93c0c13f0540 118324 lru 00000080:00200000:3.0:1788201706.977786:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f59e5647 118324: 1< 2 - - > 2< 484655104 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 69909 - -> 00000080:00200000:3.0:1788201706.977787:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000082a3768a 00000080:00200000:3.0:1788201706.977788:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201706.977789:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201706.977794:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201706.977805:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201706.977807:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201706.977808:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201706.977814:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_release():245 00000020:00010000:3.0:1788201706.977816:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201706.977817:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201706.977820:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201706.977824:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201706.977852:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3a88 (ffff93c1d864f128/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201706.977855:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f128/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201706.977859:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201706.977861:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118324, 118324] 00000080:00200000:3.0:1788201706.977872:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201706.977875:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00010000:00010000:2.0:1788201707.015226:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.015239:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.015242:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.015246:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.015249:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118500, 118500] 00000080:00200000:2.0:1788201707.015260:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.015262:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.015281:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.015284:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.015294:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485380096, count: 4096 00000080:00200000:2.0:1788201707.015300:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485380096, bytes: 4096 00000080:00200000:2.0:1788201707.015302:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.015304:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.015305:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485380096, 485384192) 0 0 00000080:00200000:2.0:1788201707.015307:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.015310:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118501, 118501] 00020000:00200000:2.0:1788201707.015312:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485380096 chunk: [485380096, 485384192) 485384192, 4096 00020000:00200000:2.0:1788201707.015313:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.015320:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118501, 118501] 00000008:00010000:2.0:1788201707.015323:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.015324:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.015330:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015335:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485380096 485384191) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015341:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015350:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015354:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.015358:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485380096, 485384192) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.015361:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118501 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.015365:0:17463:0:(rw26.c:898:ll_write_end()) pos 485380096, len 4096, copied 4096 00000080:00200000:2.0:1788201707.015367:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000033016393[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118501] 00000080:00200000:2.0:1788201707.015369:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000da616045 fffffc0001029 2:0 ffff93c0c1231e00 118501 lru 00000080:00200000:2.0:1788201707.015372:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002cc0da7d 118501: 1< 2 - - > 2< 485380096 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70086 - -> 00000080:00200000:2.0:1788201707.015373:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000033016393 00000080:00200000:2.0:1788201707.015374:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.015375:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.015381:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.015388:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.015390:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.015391:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.015398:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.015399:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.015401:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.015403:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015412:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.015416:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.015418:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.015420:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.015422:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118501, 118501] 00000080:00200000:2.0:1788201707.015428:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.015430:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.015447:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.015450:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.015460:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485384192, count: 4096 00000080:00200000:2.0:1788201707.015465:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485384192, bytes: 4096 00000080:00200000:2.0:1788201707.015467:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.015469:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.015470:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485384192, 485388288) 0 0 00000080:00200000:2.0:1788201707.015472:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.015475:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118502, 118502] 00020000:00200000:2.0:1788201707.015477:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485384192 chunk: [485384192, 485388288) 485388288, 4096 00020000:00200000:2.0:1788201707.015478:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.015485:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118502, 118502] 00000008:00010000:2.0:1788201707.015488:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.015489:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.015494:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015500:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485384192 485388287) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015509:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015513:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015517:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.015521:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485384192, 485388288) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.015524:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118502 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.015528:0:17463:0:(rw26.c:898:ll_write_end()) pos 485384192, len 4096, copied 4096 00000080:00200000:2.0:1788201707.015530:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000078eb7952[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118502] 00000080:00200000:2.0:1788201707.015532:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009bd26976 fffffc0001029 2:0 ffff93c0c12311c0 118502 lru 00000080:00200000:2.0:1788201707.015536:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008f7cdcb5 118502: 1< 2 - - > 2< 485384192 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70087 - -> 00000080:00200000:2.0:1788201707.015537:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000078eb7952 00000080:00200000:2.0:1788201707.015538:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.015539:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.015544:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.015552:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.015554:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.015555:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.015561:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.015562:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.015564:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.015567:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015572:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.015577:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.015578:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.015580:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.015582:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118502, 118502] 00000080:00200000:2.0:1788201707.015589:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.015591:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.015608:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.015610:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.015620:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485388288, count: 4096 00000080:00200000:2.0:1788201707.015626:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485388288, bytes: 4096 00000080:00200000:2.0:1788201707.015628:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.015630:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.015631:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485388288, 485392384) 0 0 00000080:00200000:2.0:1788201707.015633:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.015636:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118503, 118503] 00020000:00200000:2.0:1788201707.015637:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485388288 chunk: [485388288, 485392384) 485392384, 4096 00020000:00200000:2.0:1788201707.015639:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.015645:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118503, 118503] 00000008:00010000:2.0:1788201707.015648:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.015649:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.015654:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015664:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485388288 485392383) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015669:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015673:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015677:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.015681:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485388288, 485392384) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.015684:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118503 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.015687:0:17463:0:(rw26.c:898:ll_write_end()) pos 485388288, len 4096, copied 4096 00000080:00200000:2.0:1788201707.015689:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c033337b[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118503] 00000080:00200000:2.0:1788201707.015691:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000918948e2 fffffc0001029 2:0 ffff93c0c1231620 118503 lru 00000080:00200000:2.0:1788201707.015694:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002f7c4600 118503: 1< 2 - - > 2< 485388288 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70088 - -> 00000080:00200000:2.0:1788201707.015695:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c033337b 00000080:00200000:2.0:1788201707.015696:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.015697:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.015702:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.015710:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.015711:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.015713:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.015719:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.015720:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.015722:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.015724:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015729:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.015734:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.015735:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.015738:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.015740:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118503, 118503] 00000080:00200000:2.0:1788201707.015747:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.015748:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.015765:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.015767:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.015777:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485392384, count: 4096 00000080:00200000:2.0:1788201707.015783:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485392384, bytes: 4096 00000080:00200000:2.0:1788201707.015785:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.015787:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.015788:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485392384, 485396480) 0 0 00000080:00200000:2.0:1788201707.015790:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.015793:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118504, 118504] 00020000:00200000:2.0:1788201707.015795:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485392384 chunk: [485392384, 485396480) 485396480, 4096 00020000:00200000:2.0:1788201707.015796:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.015803:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118504, 118504] 00000008:00010000:2.0:1788201707.015806:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.015807:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.015812:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015818:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485392384 485396479) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015823:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015828:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015845:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.015850:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485392384, 485396480) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.015854:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118504 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.015857:0:17463:0:(rw26.c:898:ll_write_end()) pos 485392384, len 4096, copied 4096 00000080:00200000:2.0:1788201707.015860:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000016cc4d1e[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118504] 00000080:00200000:2.0:1788201707.015861:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000051c5cd8c fffffc0001029 2:0 ffff93c0c1231d20 118504 lru 00000080:00200000:2.0:1788201707.015865:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004a9b3d0a 118504: 1< 2 - - > 2< 485392384 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70089 - -> 00000080:00200000:2.0:1788201707.015867:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000016cc4d1e 00000080:00200000:2.0:1788201707.015867:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.015869:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.015874:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.015882:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.015884:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.015885:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.015891:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.015893:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.015894:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.015897:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015902:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.015908:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.015909:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.015911:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.015913:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118504, 118504] 00000080:00200000:2.0:1788201707.015920:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.015922:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.015939:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.015941:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.015951:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485396480, count: 4096 00000080:00200000:2.0:1788201707.015957:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485396480, bytes: 4096 00000080:00200000:2.0:1788201707.015959:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.015961:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.015962:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485396480, 485400576) 0 0 00000080:00200000:2.0:1788201707.015963:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.015967:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118505, 118505] 00020000:00200000:2.0:1788201707.015969:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485396480 chunk: [485396480, 485400576) 485400576, 4096 00020000:00200000:2.0:1788201707.015970:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.015976:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118505, 118505] 00000008:00010000:2.0:1788201707.015980:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.015981:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.015986:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015992:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485396480 485400575) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.015997:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016002:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016005:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.016024:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485396480, 485400576) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.016027:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118505 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.016031:0:17463:0:(rw26.c:898:ll_write_end()) pos 485396480, len 4096, copied 4096 00000080:00200000:2.0:1788201707.016033:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ef6575ee[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118505] 00000080:00200000:2.0:1788201707.016035:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005d018306 fffffc0001029 2:0 ffff93c0c12319a0 118505 lru 00000080:00200000:2.0:1788201707.016041:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bb330642 118505: 1< 2 - - > 2< 485396480 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70090 - -> 00000080:00200000:2.0:1788201707.016043:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ef6575ee 00000080:00200000:2.0:1788201707.016044:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.016046:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.016054:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.016071:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.016074:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.016076:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.016086:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.016088:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.016091:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.016094:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016099:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.016105:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.016106:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.016109:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.016111:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118505, 118505] 00000080:00200000:2.0:1788201707.016118:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.016120:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.016139:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.016141:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.016152:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485400576, count: 4096 00000080:00200000:2.0:1788201707.016158:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485400576, bytes: 4096 00000080:00200000:2.0:1788201707.016160:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.016162:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.016163:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485400576, 485404672) 0 0 00000080:00200000:2.0:1788201707.016164:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.016168:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118506, 118506] 00020000:00200000:2.0:1788201707.016170:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485400576 chunk: [485400576, 485404672) 485404672, 4096 00020000:00200000:2.0:1788201707.016172:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.016178:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118506, 118506] 00000008:00010000:2.0:1788201707.016182:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.016183:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.016188:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016194:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485400576 485404671) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016199:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016204:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016208:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.016212:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485400576, 485404672) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.016215:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118506 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.016219:0:17463:0:(rw26.c:898:ll_write_end()) pos 485400576, len 4096, copied 4096 00000080:00200000:2.0:1788201707.016221:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000003f9deea[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118506] 00000080:00200000:2.0:1788201707.016223:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000ab72d48 fffffc0001029 2:0 ffff93c0c1231b60 118506 lru 00000080:00200000:2.0:1788201707.016227:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004fc573b7 118506: 1< 2 - - > 2< 485400576 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70091 - -> 00000080:00200000:2.0:1788201707.016228:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000003f9deea 00000080:00200000:2.0:1788201707.016229:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.016231:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.016236:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.016243:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.016245:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.016246:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.016252:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.016254:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.016255:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.016258:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016262:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.016267:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.016268:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.016270:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.016272:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118506, 118506] 00000080:00200000:2.0:1788201707.016279:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.016281:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.016298:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.016300:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.016310:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485404672, count: 4096 00000080:00200000:2.0:1788201707.016316:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485404672, bytes: 4096 00000080:00200000:2.0:1788201707.016318:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.016320:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.016321:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485404672, 485408768) 0 0 00000080:00200000:2.0:1788201707.016323:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.016326:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118507, 118507] 00020000:00200000:2.0:1788201707.016328:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485404672 chunk: [485404672, 485408768) 485408768, 4096 00020000:00200000:2.0:1788201707.016329:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.016336:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118507, 118507] 00000008:00010000:2.0:1788201707.016341:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.016343:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.016350:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016359:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485404672 485408767) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016368:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016376:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016384:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.016392:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485404672, 485408768) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.016396:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118507 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.016400:0:17463:0:(rw26.c:898:ll_write_end()) pos 485404672, len 4096, copied 4096 00000080:00200000:2.0:1788201707.016403:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ecfe6eeb[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118507] 00000080:00200000:2.0:1788201707.016405:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000291557b fffffc0001029 2:0 ffff93c0c1231c40 118507 lru 00000080:00200000:2.0:1788201707.016408:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d5eeb1ea 118507: 1< 2 - - > 2< 485404672 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70092 - -> 00000080:00200000:2.0:1788201707.016411:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ecfe6eeb 00000080:00200000:2.0:1788201707.016412:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.016414:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.016420:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.016427:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.016429:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.016430:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.016437:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.016439:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.016440:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.016443:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016448:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.016452:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.016454:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.016456:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.016458:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118507, 118507] 00000080:00200000:2.0:1788201707.016465:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.016467:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.016485:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.016487:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.016497:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485408768, count: 4096 00000080:00200000:2.0:1788201707.016503:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485408768, bytes: 4096 00000080:00200000:2.0:1788201707.016505:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.016507:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.016508:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485408768, 485412864) 0 0 00000080:00200000:2.0:1788201707.016510:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.016513:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118508, 118508] 00020000:00200000:2.0:1788201707.016515:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485408768 chunk: [485408768, 485412864) 485412864, 4096 00020000:00200000:2.0:1788201707.016516:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.016523:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118508, 118508] 00000008:00010000:2.0:1788201707.016526:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.016527:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.016532:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016538:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485408768 485412863) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016544:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016548:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016552:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.016556:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485408768, 485412864) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.016559:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118508 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.016562:0:17463:0:(rw26.c:898:ll_write_end()) pos 485408768, len 4096, copied 4096 00000080:00200000:2.0:1788201707.016565:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f0653685[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118508] 00000080:00200000:2.0:1788201707.016567:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e17233a7 fffffc0001029 2:0 ffff93c0c1231540 118508 lru 00000080:00200000:2.0:1788201707.016571:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000007e2fc24 118508: 1< 2 - - > 2< 485408768 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70093 - -> 00000080:00200000:2.0:1788201707.016572:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f0653685 00000080:00200000:2.0:1788201707.016573:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.016575:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.016580:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.016587:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.016588:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.016590:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.016596:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.016597:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.016599:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.016601:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016606:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.016611:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.016612:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.016615:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.016617:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118508, 118508] 00000080:00200000:2.0:1788201707.016623:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.016625:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.016642:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.016644:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.016654:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485412864, count: 4096 00000080:00200000:2.0:1788201707.016660:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485412864, bytes: 4096 00000080:00200000:2.0:1788201707.016662:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.016664:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.016665:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485412864, 485416960) 0 0 00000080:00200000:2.0:1788201707.016667:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.016670:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118509, 118509] 00020000:00200000:2.0:1788201707.016672:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485412864 chunk: [485412864, 485416960) 485416960, 4096 00020000:00200000:2.0:1788201707.016673:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.016679:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118509, 118509] 00000008:00010000:2.0:1788201707.016682:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.016684:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.016689:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016694:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485412864 485416959) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016700:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016704:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016708:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.016712:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485412864, 485416960) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.016715:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118509 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.016719:0:17463:0:(rw26.c:898:ll_write_end()) pos 485412864, len 4096, copied 4096 00000080:00200000:2.0:1788201707.016721:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000096e8a1d1[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118509] 00000080:00200000:2.0:1788201707.016723:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008d9b9831 fffffc0001029 2:0 ffff93c0c4415b60 118509 lru 00000080:00200000:2.0:1788201707.016726:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ed874da1 118509: 1< 2 - - > 2< 485412864 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70094 - -> 00000080:00200000:2.0:1788201707.016728:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000096e8a1d1 00000080:00200000:2.0:1788201707.016729:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.016730:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.016735:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.016743:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.016744:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.016746:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.016751:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.016753:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.016754:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.016757:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016762:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.016766:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.016768:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.016770:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.016772:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118509, 118509] 00000080:00200000:2.0:1788201707.016778:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.016780:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.016797:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.016800:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.016810:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485416960, count: 4096 00000080:00200000:2.0:1788201707.016816:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485416960, bytes: 4096 00000080:00200000:2.0:1788201707.016817:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.016819:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.016821:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485416960, 485421056) 0 0 00000080:00200000:2.0:1788201707.016822:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.016825:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118510, 118510] 00020000:00200000:2.0:1788201707.016827:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485416960 chunk: [485416960, 485421056) 485421056, 4096 00020000:00200000:2.0:1788201707.016841:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.016849:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118510, 118510] 00000008:00010000:2.0:1788201707.016852:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.016854:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.016859:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016864:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485416960 485421055) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016870:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016874:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016878:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.016883:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485416960, 485421056) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.016886:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118510 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.016890:0:17463:0:(rw26.c:898:ll_write_end()) pos 485416960, len 4096, copied 4096 00000080:00200000:2.0:1788201707.016892:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e02b15b4[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118510] 00000080:00200000:2.0:1788201707.016894:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ee7f99bd fffffc0001029 2:0 ffff93c0c4415c40 118510 lru 00000080:00200000:2.0:1788201707.016898:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000049412861 118510: 1< 2 - - > 2< 485416960 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70095 - -> 00000080:00200000:2.0:1788201707.016899:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e02b15b4 00000080:00200000:2.0:1788201707.016900:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.016901:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.016906:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.016914:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.016916:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.016917:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.016923:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.016924:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.016926:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.016929:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.016934:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.016938:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.016940:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.016942:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.016944:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118510, 118510] 00000080:00200000:2.0:1788201707.016950:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.016952:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.016976:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.016980:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.016996:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485421056, count: 4096 00000080:00200000:2.0:1788201707.017006:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485421056, bytes: 4096 00000080:00200000:2.0:1788201707.017019:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.017024:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.017025:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485421056, 485425152) 0 0 00000080:00200000:2.0:1788201707.017028:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.017033:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118511, 118511] 00020000:00200000:2.0:1788201707.017036:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485421056 chunk: [485421056, 485425152) 485425152, 4096 00020000:00200000:2.0:1788201707.017038:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.017050:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118511, 118511] 00000008:00010000:2.0:1788201707.017055:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.017057:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.017064:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017073:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485421056 485425151) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017083:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017089:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017095:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.017107:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485421056, 485425152) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.017112:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118511 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.017118:0:17463:0:(rw26.c:898:ll_write_end()) pos 485421056, len 4096, copied 4096 00000080:00200000:2.0:1788201707.017122:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001524a932[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118511] 00000080:00200000:2.0:1788201707.017125:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000090590bef fffffc0001029 2:0 ffff93c0c4415540 118511 lru 00000080:00200000:2.0:1788201707.017131:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000086e8f9a8 118511: 1< 2 - - > 2< 485421056 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70096 - -> 00000080:00200000:2.0:1788201707.017134:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001524a932 00000080:00200000:2.0:1788201707.017135:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.017137:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.017145:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.017157:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.017160:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.017162:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.017170:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.017172:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.017174:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.017177:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017182:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.017187:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.017188:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.017192:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.017194:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118511, 118511] 00000080:00200000:2.0:1788201707.017201:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.017203:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.017221:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.017223:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.017234:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485425152, count: 4096 00000080:00200000:2.0:1788201707.017240:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485425152, bytes: 4096 00000080:00200000:2.0:1788201707.017242:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.017244:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.017245:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485425152, 485429248) 0 0 00000080:00200000:2.0:1788201707.017247:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.017250:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118512, 118512] 00020000:00200000:2.0:1788201707.017252:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485425152 chunk: [485425152, 485429248) 485429248, 4096 00020000:00200000:2.0:1788201707.017253:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.017260:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118512, 118512] 00000008:00010000:2.0:1788201707.017263:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.017264:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.017269:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017275:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485425152 485429247) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017280:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017285:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017292:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.017297:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485425152, 485429248) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.017299:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118512 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.017303:0:17463:0:(rw26.c:898:ll_write_end()) pos 485425152, len 4096, copied 4096 00000080:00200000:2.0:1788201707.017305:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d1e774c0[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118512] 00000080:00200000:2.0:1788201707.017307:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000028bcd979 fffffc0001029 2:0 ffff93c0c4415460 118512 lru 00000080:00200000:2.0:1788201707.017311:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000096cf42e5 118512: 1< 2 - - > 2< 485425152 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70097 - -> 00000080:00200000:2.0:1788201707.017312:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d1e774c0 00000080:00200000:2.0:1788201707.017313:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.017314:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.017319:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.017327:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.017329:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.017331:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.017339:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.017342:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.017344:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.017348:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017356:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.017369:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.017372:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.017376:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.017380:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118512, 118512] 00000080:00200000:2.0:1788201707.017388:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.017391:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.017410:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.017412:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.017422:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485429248, count: 4096 00000080:00200000:2.0:1788201707.017428:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485429248, bytes: 4096 00000080:00200000:2.0:1788201707.017430:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.017432:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.017433:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485429248, 485433344) 0 0 00000080:00200000:2.0:1788201707.017434:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.017438:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118513, 118513] 00020000:00200000:2.0:1788201707.017440:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485429248 chunk: [485429248, 485433344) 485433344, 4096 00020000:00200000:2.0:1788201707.017441:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.017448:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118513, 118513] 00000008:00010000:2.0:1788201707.017451:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.017452:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.017458:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017463:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485429248 485433343) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017469:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017477:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017482:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.017486:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485429248, 485433344) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.017488:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118513 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.017492:0:17463:0:(rw26.c:898:ll_write_end()) pos 485429248, len 4096, copied 4096 00000080:00200000:2.0:1788201707.017495:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c6315169[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118513] 00000080:00200000:2.0:1788201707.017496:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000085036484 fffffc0001029 2:0 ffff93c0c44150e0 118513 lru 00000080:00200000:2.0:1788201707.017500:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000022d9a291 118513: 1< 2 - - > 2< 485429248 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70098 - -> 00000080:00200000:2.0:1788201707.017501:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c6315169 00000080:00200000:2.0:1788201707.017501:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.017503:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.017508:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.017516:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.017518:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.017519:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.017525:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.017527:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.017528:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.017530:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017538:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.017543:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.017544:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.017547:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.017549:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118513, 118513] 00000080:00200000:2.0:1788201707.017555:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.017557:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.017574:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.017576:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.017586:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485433344, count: 4096 00000080:00200000:2.0:1788201707.017592:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485433344, bytes: 4096 00000080:00200000:2.0:1788201707.017594:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.017596:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.017597:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485433344, 485437440) 0 0 00000080:00200000:2.0:1788201707.017598:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.017601:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118514, 118514] 00020000:00200000:2.0:1788201707.017603:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485433344 chunk: [485433344, 485437440) 485437440, 4096 00020000:00200000:2.0:1788201707.017605:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.017615:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118514, 118514] 00000008:00010000:2.0:1788201707.017619:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.017622:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.017629:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017638:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485433344 485437439) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017651:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017658:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017665:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.017671:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485433344, 485437440) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.017677:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118514 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.017682:0:17463:0:(rw26.c:898:ll_write_end()) pos 485433344, len 4096, copied 4096 00000080:00200000:2.0:1788201707.017687:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000008cd1a8e[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118514] 00000080:00200000:2.0:1788201707.017690:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000040cff4ff fffffc0001029 2:0 ffff93c0c4415700 118514 lru 00000080:00200000:2.0:1788201707.017695:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f7dfddf4 118514: 1< 2 - - > 2< 485433344 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70099 - -> 00000080:00200000:2.0:1788201707.017697:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000008cd1a8e 00000080:00200000:2.0:1788201707.017698:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.017700:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.017710:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.017722:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.017724:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.017726:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.017736:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.017739:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.017741:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.017745:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017753:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.017762:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.017764:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.017769:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.017772:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118514, 118514] 00000080:00200000:2.0:1788201707.017783:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.017785:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.017813:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.017816:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.017848:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485437440, count: 4096 00000080:00200000:2.0:1788201707.017860:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485437440, bytes: 4096 00000080:00200000:2.0:1788201707.017863:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.017867:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.017868:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485437440, 485441536) 0 0 00000080:00200000:2.0:1788201707.017870:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.017876:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118515, 118515] 00020000:00200000:2.0:1788201707.017880:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485437440 chunk: [485437440, 485441536) 485441536, 4096 00020000:00200000:2.0:1788201707.017882:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.017894:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118515, 118515] 00000008:00010000:2.0:1788201707.017900:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.017903:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.017910:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017924:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485437440 485441535) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017933:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017941:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.017949:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.017956:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485437440, 485441536) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.017962:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118515 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.017967:0:17463:0:(rw26.c:898:ll_write_end()) pos 485437440, len 4096, copied 4096 00000080:00200000:2.0:1788201707.017971:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008e8b4ed9[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118515] 00000080:00200000:2.0:1788201707.017974:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000615c6460 fffffc0001029 2:0 ffff93c0c4415000 118515 lru 00000080:00200000:2.0:1788201707.017979:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003d38a422 118515: 1< 2 - - > 2< 485437440 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70100 - -> 00000080:00200000:2.0:1788201707.017982:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008e8b4ed9 00000080:00200000:2.0:1788201707.017983:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.017985:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.017994:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.018006:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.018022:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.018025:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.018035:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.018038:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.018040:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.018044:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018053:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.018061:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.018064:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.018068:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.018071:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118515, 118515] 00000080:00200000:2.0:1788201707.018082:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.018085:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.018114:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.018118:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.018133:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485441536, count: 4096 00000080:00200000:2.0:1788201707.018145:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485441536, bytes: 4096 00000080:00200000:2.0:1788201707.018148:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.018152:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.018153:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485441536, 485445632) 0 0 00000080:00200000:2.0:1788201707.018156:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.018160:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118516, 118516] 00020000:00200000:2.0:1788201707.018163:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485441536 chunk: [485441536, 485445632) 485445632, 4096 00020000:00200000:2.0:1788201707.018166:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.018178:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118516, 118516] 00000008:00010000:2.0:1788201707.018183:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.018186:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.018194:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018203:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485441536 485445631) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018212:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018220:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018226:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.018234:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485441536, 485445632) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.018239:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118516 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.018244:0:17463:0:(rw26.c:898:ll_write_end()) pos 485441536, len 4096, copied 4096 00000080:00200000:2.0:1788201707.018248:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ba6ed68b[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118516] 00000080:00200000:2.0:1788201707.018251:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a44aae08 fffffc0001029 2:0 ffff93c0c4415a80 118516 lru 00000080:00200000:2.0:1788201707.018256:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000044407f4c 118516: 1< 2 - - > 2< 485441536 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70101 - -> 00000080:00200000:2.0:1788201707.018258:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ba6ed68b 00000080:00200000:2.0:1788201707.018259:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.018261:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.018269:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.018281:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.018284:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.018286:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.018294:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.018297:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.018300:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.018304:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018312:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.018320:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.018322:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.018326:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.018329:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118516, 118516] 00000080:00200000:2.0:1788201707.018340:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.018343:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.018371:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.018375:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.018389:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485445632, count: 4096 00000080:00200000:2.0:1788201707.018400:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485445632, bytes: 4096 00000080:00200000:2.0:1788201707.018402:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.018406:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.018408:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485445632, 485449728) 0 0 00000080:00200000:2.0:1788201707.018410:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.018415:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118517, 118517] 00020000:00200000:2.0:1788201707.018418:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485445632 chunk: [485445632, 485449728) 485449728, 4096 00020000:00200000:2.0:1788201707.018420:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.018431:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118517, 118517] 00000008:00010000:2.0:1788201707.018437:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.018439:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.018447:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018456:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485445632 485449727) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018464:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018472:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018479:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.018486:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485445632, 485449728) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.018491:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118517 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.018497:0:17463:0:(rw26.c:898:ll_write_end()) pos 485445632, len 4096, copied 4096 00000080:00200000:2.0:1788201707.018500:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003c1f9e13[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118517] 00000080:00200000:2.0:1788201707.018503:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d6f13b23 fffffc0001029 2:0 ffff93c0c4415380 118517 lru 00000080:00200000:2.0:1788201707.018509:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f176009a 118517: 1< 2 - - > 2< 485445632 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70102 - -> 00000080:00200000:2.0:1788201707.018511:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003c1f9e13 00000080:00200000:2.0:1788201707.018512:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.018514:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.018522:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.018538:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.018541:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.018543:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.018552:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.018555:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.018557:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.018561:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018569:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.018577:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.018580:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.018584:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.018587:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118517, 118517] 00000080:00200000:2.0:1788201707.018598:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.018601:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.018627:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.018631:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.018646:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485449728, count: 4096 00000080:00200000:2.0:1788201707.018656:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485449728, bytes: 4096 00000080:00200000:2.0:1788201707.018659:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.018662:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.018663:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485449728, 485453824) 0 0 00000080:00200000:2.0:1788201707.018666:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.018670:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118518, 118518] 00020000:00200000:2.0:1788201707.018673:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485449728 chunk: [485449728, 485453824) 485453824, 4096 00020000:00200000:2.0:1788201707.018675:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.018686:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118518, 118518] 00000008:00010000:2.0:1788201707.018691:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.018694:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.018701:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018710:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485449728 485453823) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018718:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018726:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018733:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.018741:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485449728, 485453824) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.018746:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118518 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.018751:0:17463:0:(rw26.c:898:ll_write_end()) pos 485449728, len 4096, copied 4096 00000080:00200000:2.0:1788201707.018755:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ecead7c0[3 0000000075f4bf38 2 1 0000000058ed61ac 0 0 118518] 00000080:00200000:2.0:1788201707.018758:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000456c8afa fffffc0001029 2:0 ffff93c0c44158c0 118518 lru 00000080:00200000:2.0:1788201707.018764:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000076ab7567 118518: 1< 2 - - > 2< 485449728 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70103 - -> 00000080:00200000:2.0:1788201707.018766:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ecead7c0 00000080:00200000:2.0:1788201707.018767:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.018770:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.018778:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.018789:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.018792:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.018794:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.018802:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.018806:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.018808:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.018812:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.018821:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.018855:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702f38 (ffff93c1d300a3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.018858:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.018863:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.018866:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118518, 118518] 00000080:00200000:2.0:1788201707.018877:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.018880:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.018951:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.018956:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.018977:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485453824, count: 4096 00000080:00200000:3.0:1788201707.018995:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485453824, bytes: 4096 00000080:00200000:3.0:1788201707.018999:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.019003:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.019005:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485453824, 485457920) 0 0 00000080:00200000:3.0:1788201707.019022:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.019029:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118519, 118519] 00020000:00200000:3.0:1788201707.019034:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485453824 chunk: [485453824, 485457920) 485457920, 4096 00020000:00200000:3.0:1788201707.019042:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.019065:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118519, 118519] 00000008:00010000:3.0:1788201707.019072:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.019075:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.019085:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.019096:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485453824 485457919) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.019106:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.019114:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.019121:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.019130:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485453824, 485457920) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.019136:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118519 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.019142:0:17463:0:(rw26.c:898:ll_write_end()) pos 485453824, len 4096, copied 4096 00000080:00200000:3.0:1788201707.019147:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002f0d61b6[3 0000000075f4bf38 2 1 0000000067111b74 0 0 118519] 00000080:00200000:3.0:1788201707.019151:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008b6c8755 fffffc0001029 2:0 ffff93c0c4415ee0 118519 lru 00000080:00200000:3.0:1788201707.019158:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e6c394c8 118519: 1< 2 - - > 2< 485453824 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70104 - -> 00000080:00200000:3.0:1788201707.019160:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002f0d61b6 00000080:00200000:3.0:1788201707.019161:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.019164:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.019174:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.019188:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.019191:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.019193:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.019205:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3218 (ffff93c1d864f368/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.019208:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3218 (ffff93c1d864f368/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.019211:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.019225:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.019233:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.019243:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3218 (ffff93c1d864f368/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.019246:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.019250:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.019254:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118519, 118519] 00000080:00200000:3.0:1788201707.019272:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.019276:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.019319:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.019323:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.019351:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485457920, count: 4096 00000080:00200000:3.0:1788201707.019363:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485457920, bytes: 4096 00000080:00200000:3.0:1788201707.019367:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.019372:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.019374:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485457920, 485462016) 0 0 00000080:00200000:3.0:1788201707.019377:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.019382:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118520, 118520] 00020000:00200000:3.0:1788201707.019385:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485457920 chunk: [485457920, 485462016) 485462016, 4096 00020000:00200000:3.0:1788201707.019388:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.019401:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118520, 118520] 00000008:00010000:3.0:1788201707.019406:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.019409:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.019417:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.019427:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485457920 485462015) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.019436:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.019444:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.019452:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.019460:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485457920, 485462016) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.019466:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118520 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.019471:0:17463:0:(rw26.c:898:ll_write_end()) pos 485457920, len 4096, copied 4096 00000080:00200000:3.0:1788201707.019476:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000061f82d79[3 0000000075f4bf38 2 1 0000000067111b74 0 0 118520] 00000080:00200000:3.0:1788201707.019479:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000695304f fffffc0001029 2:0 ffff93c0c44157e0 118520 lru 00000080:00200000:3.0:1788201707.019485:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003a836870 118520: 1< 2 - - > 2< 485457920 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70105 - -> 00000080:00200000:3.0:1788201707.019488:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000061f82d79 00000080:00200000:3.0:1788201707.019489:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.019492:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.019501:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.019513:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.019516:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.019518:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.019528:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3218 (ffff93c1d864f368/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.019531:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3218 (ffff93c1d864f368/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.019534:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.019537:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.019545:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.019553:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3218 (ffff93c1d864f368/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.019556:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.019560:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.019563:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118520, 118520] 00000080:00200000:3.0:1788201707.019575:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.019579:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.019610:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.019613:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.019631:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485462016, count: 4096 00000080:00200000:3.0:1788201707.019640:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485462016, bytes: 4096 00000080:00200000:3.0:1788201707.019642:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.019645:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.019646:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485462016, 485466112) 0 0 00000080:00200000:3.0:1788201707.019648:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.019652:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118521, 118521] 00020000:00200000:3.0:1788201707.019654:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485462016 chunk: [485462016, 485466112) 485466112, 4096 00020000:00200000:3.0:1788201707.019656:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.019664:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118521, 118521] 00000008:00010000:3.0:1788201707.019668:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.019670:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.019678:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.019687:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485462016 485466111) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.019696:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.019703:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.019709:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.019717:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485462016, 485466112) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.019723:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118521 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.019729:0:17463:0:(rw26.c:898:ll_write_end()) pos 485462016, len 4096, copied 4096 00000080:00200000:3.0:1788201707.019733:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000042c98889[3 0000000075f4bf38 2 1 0000000067111b74 0 0 118521] 00000080:00200000:3.0:1788201707.019737:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000062b13a6 fffffc0001029 2:0 ffff93c0c44152a0 118521 lru 00000080:00200000:3.0:1788201707.019743:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a983ab64 118521: 1< 2 - - > 2< 485462016 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70106 - -> 00000080:00200000:3.0:1788201707.019745:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000042c98889 00000080:00200000:3.0:1788201707.019746:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.019748:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.019756:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.019769:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.019772:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.019774:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.019783:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3218 (ffff93c1d864f368/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.019786:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3218 (ffff93c1d864f368/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.019789:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.019792:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.019800:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.019808:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3218 (ffff93c1d864f368/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.019810:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f368/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.019814:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.019817:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118521, 118521] 00000080:00200000:3.0:1788201707.019853:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.019857:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.019901:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.019906:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.019925:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485466112, count: 4096 00000080:00200000:2.0:1788201707.019935:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485466112, bytes: 4096 00000080:00200000:2.0:1788201707.019938:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.019941:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.019943:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485466112, 485470208) 0 0 00000080:00200000:2.0:1788201707.019946:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.019951:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118522, 118522] 00020000:00200000:2.0:1788201707.019953:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485466112 chunk: [485466112, 485470208) 485470208, 4096 00020000:00200000:2.0:1788201707.019955:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.019966:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118522, 118522] 00000008:00010000:2.0:1788201707.019970:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.019972:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.019979:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.019987:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485466112 485470207) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.019993:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.019998:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020003:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.020007:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485466112, 485470208) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.020025:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118522 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.020029:0:17463:0:(rw26.c:898:ll_write_end()) pos 485466112, len 4096, copied 4096 00000080:00200000:2.0:1788201707.020032:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005180ad64[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118522] 00000080:00200000:2.0:1788201707.020034:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e91b11a8 fffffc0001029 2:0 ffff93c0c4415e00 118522 lru 00000080:00200000:2.0:1788201707.020038:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000eca8bed8 118522: 1< 2 - - > 2< 485466112 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70107 - -> 00000080:00200000:2.0:1788201707.020040:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005180ad64 00000080:00200000:2.0:1788201707.020041:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.020043:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.020050:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.020059:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.020061:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.020063:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.020071:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.020073:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.020075:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.020077:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020083:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.020088:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.020089:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.020092:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.020094:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118522, 118522] 00000080:00200000:2.0:1788201707.020102:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.020104:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.020124:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.020126:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.020137:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485470208, count: 4096 00000080:00200000:2.0:1788201707.020144:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485470208, bytes: 4096 00000080:00200000:2.0:1788201707.020146:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.020148:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.020150:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485470208, 485474304) 0 0 00000080:00200000:2.0:1788201707.020151:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.020155:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118523, 118523] 00020000:00200000:2.0:1788201707.020157:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485470208 chunk: [485470208, 485474304) 485474304, 4096 00020000:00200000:2.0:1788201707.020158:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.020166:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118523, 118523] 00000008:00010000:2.0:1788201707.020169:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.020171:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.020188:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020198:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485470208 485474303) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020207:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020215:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020223:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.020231:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485470208, 485474304) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.020237:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118523 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.020241:0:17463:0:(rw26.c:898:ll_write_end()) pos 485470208, len 4096, copied 4096 00000080:00200000:2.0:1788201707.020244:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005750091e[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118523] 00000080:00200000:2.0:1788201707.020247:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f694ee56 fffffc0001029 2:0 ffff93c0c44151c0 118523 lru 00000080:00200000:2.0:1788201707.020251:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003f78a9a2 118523: 1< 2 - - > 2< 485470208 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70108 - -> 00000080:00200000:2.0:1788201707.020253:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005750091e 00000080:00200000:2.0:1788201707.020254:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.020255:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.020262:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.020280:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.020283:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.020286:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.020297:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.020299:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.020302:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.020306:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020314:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.020322:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.020325:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.020329:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.020332:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118523, 118523] 00000080:00200000:2.0:1788201707.020343:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.020346:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.020375:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.020379:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.020396:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485474304, count: 4096 00000080:00200000:2.0:1788201707.020406:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485474304, bytes: 4096 00000080:00200000:2.0:1788201707.020409:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.020413:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.020415:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485474304, 485478400) 0 0 00000080:00200000:2.0:1788201707.020417:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.020422:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118524, 118524] 00020000:00200000:2.0:1788201707.020425:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485474304 chunk: [485474304, 485478400) 485478400, 4096 00020000:00200000:2.0:1788201707.020427:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.020438:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118524, 118524] 00000008:00010000:2.0:1788201707.020443:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.020445:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.020453:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020459:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485474304 485478399) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020465:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020470:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020474:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.020479:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485474304, 485478400) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.020483:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118524 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.020487:0:17463:0:(rw26.c:898:ll_write_end()) pos 485474304, len 4096, copied 4096 00000080:00200000:2.0:1788201707.020490:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d412732d[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118524] 00000080:00200000:2.0:1788201707.020492:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000038e9420f fffffc0001029 2:0 ffff93c0c4415620 118524 lru 00000080:00200000:2.0:1788201707.020496:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002081479a 118524: 1< 2 - - > 2< 485474304 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70109 - -> 00000080:00200000:2.0:1788201707.020497:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d412732d 00000080:00200000:2.0:1788201707.020498:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.020500:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.020506:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.020514:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.020516:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.020518:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.020524:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.020526:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.020528:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.020530:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020535:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.020541:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.020542:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.020545:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.020546:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118524, 118524] 00000080:00200000:2.0:1788201707.020553:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.020555:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.020574:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.020577:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.020587:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485478400, count: 4096 00000080:00200000:2.0:1788201707.020593:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485478400, bytes: 4096 00000080:00200000:2.0:1788201707.020595:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.020597:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.020598:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485478400, 485482496) 0 0 00000080:00200000:2.0:1788201707.020600:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.020604:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118525, 118525] 00020000:00200000:2.0:1788201707.020606:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485478400 chunk: [485478400, 485482496) 485482496, 4096 00020000:00200000:2.0:1788201707.020607:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.020614:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118525, 118525] 00000008:00010000:2.0:1788201707.020617:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.020618:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.020623:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020629:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485478400 485482495) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020635:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020639:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020643:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.020648:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485478400, 485482496) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.020652:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118525 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.020656:0:17463:0:(rw26.c:898:ll_write_end()) pos 485478400, len 4096, copied 4096 00000080:00200000:2.0:1788201707.020658:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003e2a798e[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118525] 00000080:00200000:2.0:1788201707.020660:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000371ab191 fffffc0001029 2:0 ffff93c0c4415d20 118525 lru 00000080:00200000:2.0:1788201707.020664:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d4bfaa34 118525: 1< 2 - - > 2< 485478400 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70110 - -> 00000080:00200000:2.0:1788201707.020666:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003e2a798e 00000080:00200000:2.0:1788201707.020666:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.020668:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.020673:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.020681:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.020683:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.020684:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.020690:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.020692:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.020693:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.020696:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020700:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.020705:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.020706:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.020709:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.020711:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118525, 118525] 00000080:00200000:2.0:1788201707.020717:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.020719:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.020737:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.020739:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.020749:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485482496, count: 4096 00000080:00200000:2.0:1788201707.020755:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485482496, bytes: 4096 00000080:00200000:2.0:1788201707.020757:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.020759:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.020760:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485482496, 485486592) 0 0 00000080:00200000:2.0:1788201707.020762:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.020765:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118526, 118526] 00020000:00200000:2.0:1788201707.020767:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485482496 chunk: [485482496, 485486592) 485486592, 4096 00020000:00200000:2.0:1788201707.020768:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.020774:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118526, 118526] 00000008:00010000:2.0:1788201707.020777:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.020779:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.020784:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020793:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485482496 485486591) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020803:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020811:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020818:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.020848:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485482496, 485486592) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.020852:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118526 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.020856:0:17463:0:(rw26.c:898:ll_write_end()) pos 485482496, len 4096, copied 4096 00000080:00200000:2.0:1788201707.020859:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004a3eaffd[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118526] 00000080:00200000:2.0:1788201707.020861:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d5e32611 fffffc0001029 2:0 ffff93c0c44159a0 118526 lru 00000080:00200000:2.0:1788201707.020865:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000015276e79 118526: 1< 2 - - > 2< 485482496 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70111 - -> 00000080:00200000:2.0:1788201707.020866:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004a3eaffd 00000080:00200000:2.0:1788201707.020867:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.020868:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.020874:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.020882:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.020884:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.020885:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.020892:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.020893:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.020895:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.020898:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020903:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.020908:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.020909:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.020912:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.020914:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118526, 118526] 00000080:00200000:2.0:1788201707.020921:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.020923:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.020941:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.020943:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.020954:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485486592, count: 4096 00000080:00200000:2.0:1788201707.020959:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485486592, bytes: 4096 00000080:00200000:2.0:1788201707.020961:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.020964:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.020965:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485486592, 485490688) 0 0 00000080:00200000:2.0:1788201707.020966:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.020969:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118527, 118527] 00020000:00200000:2.0:1788201707.020971:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485486592 chunk: [485486592, 485490688) 485490688, 4096 00020000:00200000:2.0:1788201707.020973:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.020979:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118527, 118527] 00000008:00010000:2.0:1788201707.020983:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.020984:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.020989:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.020995:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485486592 485490687) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021000:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021005:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021026:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.021032:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485486592, 485490688) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.021036:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118527 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.021039:0:17463:0:(rw26.c:898:ll_write_end()) pos 485486592, len 4096, copied 4096 00000080:00200000:2.0:1788201707.021042:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000622f820e[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118527] 00000080:00200000:2.0:1788201707.021044:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000079e9a81f fffffc0001029 2:0 ffff93c0c44167e0 118527 lru 00000080:00200000:2.0:1788201707.021048:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000097eacc25 118527: 1< 2 - - > 2< 485486592 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70112 - -> 00000080:00200000:2.0:1788201707.021049:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000622f820e 00000080:00200000:2.0:1788201707.021050:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.021051:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.021057:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 000000006786f87e ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.021065:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.021066:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.021068:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.021075:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.021076:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.021078:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.021080:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021085:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.021094:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.021095:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.021098:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.021100:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118527, 118527] 00000080:00200000:2.0:1788201707.021107:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.021109:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.021127:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.021129:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.021139:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485490688, count: 4096 00000080:00200000:2.0:1788201707.021144:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485490688, bytes: 4096 00000080:00200000:2.0:1788201707.021146:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.021149:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.021150:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485490688, 485494784) 0 0 00000080:00200000:2.0:1788201707.021151:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.021154:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118528, 118528] 00020000:00200000:2.0:1788201707.021156:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485490688 chunk: [485490688, 485494784) 485494784, 4096 00020000:00200000:2.0:1788201707.021158:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.021165:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118528, 118528] 00000008:00010000:2.0:1788201707.021168:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.021169:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.021174:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021180:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485490688 485494783) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 285/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021186:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021199:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021206:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.021214:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485490688, 485494784) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.021219:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118528 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.021224:0:17463:0:(rw26.c:898:ll_write_end()) pos 485490688, len 4096, copied 4096 00000080:00200000:2.0:1788201707.021228:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000b18fcca[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118528] 00000080:00200000:2.0:1788201707.021232:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000093fa7736 fffffc0001029 2:0 ffff93c0c44162a0 118528 lru 00000080:00200000:2.0:1788201707.021238:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a7a9d3ec 118528: 1< 2 - - > 2< 485490688 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 5111808 - | + - + - > 5< + - + - | 0 - | 70113 - -> 00000080:00200000:2.0:1788201707.021240:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000b18fcca 00000080:00200000:2.0:1788201707.021241:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.021242:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.021250:0:17463:0:(osc_cache.c:857:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.021258:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.021259:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.021261:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.021267:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.021269:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.021270:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.021273:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021281:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.021287:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.021289:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.021293:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.021296:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118528, 118528] 00000080:00200000:2.0:1788201707.021306:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.021310:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.021336:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.021340:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.021352:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485494784, count: 4096 00000080:00200000:2.0:1788201707.021359:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485494784, bytes: 4096 00000080:00200000:2.0:1788201707.021361:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.021363:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.021364:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485494784, 485498880) 0 0 00000080:00200000:2.0:1788201707.021366:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.021370:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118529, 118529] 00020000:00200000:2.0:1788201707.021372:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485494784 chunk: [485494784, 485498880) 485498880, 4096 00020000:00200000:2.0:1788201707.021373:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.021380:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118529, 118529] 00000008:00010000:2.0:1788201707.021383:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.021385:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.021390:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021396:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485494784 485498879) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021406:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021410:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021414:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.021419:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485494784, 485498880) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.021422:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118529 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.021426:0:17463:0:(rw26.c:898:ll_write_end()) pos 485494784, len 4096, copied 4096 00000080:00200000:2.0:1788201707.021428:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009f29387e[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118529] 00000080:00200000:2.0:1788201707.021430:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000510087fe fffffc0001029 2:0 ffff93c0c4416e00 118529 lru 00000080:00200000:2.0:1788201707.021434:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ba0c72e1 118529: 1< 2 - - > 2< 485494784 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70114 - -> 00000080:00200000:2.0:1788201707.021435:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009f29387e 00000080:00200000:2.0:1788201707.021436:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.021437:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.021443:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.021451:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.021452:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.021453:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.021460:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.021461:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.021463:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.021465:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021470:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.021475:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.021477:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.021479:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.021481:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118529, 118529] 00000080:00200000:2.0:1788201707.021488:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.021490:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.021507:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.021509:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.021519:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485498880, count: 4096 00000080:00200000:2.0:1788201707.021525:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485498880, bytes: 4096 00000080:00200000:2.0:1788201707.021527:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.021529:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.021530:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485498880, 485502976) 0 0 00000080:00200000:2.0:1788201707.021532:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.021535:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118530, 118530] 00020000:00200000:2.0:1788201707.021537:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485498880 chunk: [485498880, 485502976) 485502976, 4096 00020000:00200000:2.0:1788201707.021538:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.021545:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118530, 118530] 00000008:00010000:2.0:1788201707.021548:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.021549:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.021554:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021563:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485498880 485502975) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021569:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021573:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021577:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.021581:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485498880, 485502976) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.021583:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118530 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.021587:0:17463:0:(rw26.c:898:ll_write_end()) pos 485498880, len 4096, copied 4096 00000080:00200000:2.0:1788201707.021589:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008a59dfb9[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118530] 00000080:00200000:2.0:1788201707.021591:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009bca4681 fffffc0001029 2:0 ffff93c0c44161c0 118530 lru 00000080:00200000:2.0:1788201707.021594:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008f6a5175 118530: 1< 2 - - > 2< 485498880 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70115 - -> 00000080:00200000:2.0:1788201707.021596:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008a59dfb9 00000080:00200000:2.0:1788201707.021597:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.021598:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.021603:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.021610:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.021612:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.021613:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.021619:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.021621:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.021622:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.021625:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021630:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.021635:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.021636:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.021638:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.021640:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118530, 118530] 00000080:00200000:2.0:1788201707.021646:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.021648:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.021666:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.021668:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.021678:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485502976, count: 4096 00000080:00200000:2.0:1788201707.021683:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485502976, bytes: 4096 00000080:00200000:2.0:1788201707.021685:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.021687:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.021688:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485502976, 485507072) 0 0 00000080:00200000:2.0:1788201707.021690:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.021693:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118531, 118531] 00020000:00200000:2.0:1788201707.021695:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485502976 chunk: [485502976, 485507072) 485507072, 4096 00020000:00200000:2.0:1788201707.021696:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.021703:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118531, 118531] 00000008:00010000:2.0:1788201707.021706:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.021707:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.021713:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021719:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485502976 485507071) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021724:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021728:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021732:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.021736:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485502976, 485507072) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.021739:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118531 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.021742:0:17463:0:(rw26.c:898:ll_write_end()) pos 485502976, len 4096, copied 4096 00000080:00200000:2.0:1788201707.021744:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000090edaba7[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118531] 00000080:00200000:2.0:1788201707.021746:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000009fc184 fffffc0001029 2:0 ffff93c0c4416620 118531 lru 00000080:00200000:2.0:1788201707.021750:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000561fbd61 118531: 1< 2 - - > 2< 485502976 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70116 - -> 00000080:00200000:2.0:1788201707.021751:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000090edaba7 00000080:00200000:2.0:1788201707.021751:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.021752:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.021757:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.021765:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.021767:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.021769:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.021774:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.021776:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.021778:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.021780:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021785:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.021794:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.021796:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.021800:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.021803:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118531, 118531] 00000080:00200000:2.0:1788201707.021813:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.021816:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.021863:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.021866:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.021877:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485507072, count: 4096 00000080:00200000:2.0:1788201707.021883:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485507072, bytes: 4096 00000080:00200000:2.0:1788201707.021884:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.021887:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.021888:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485507072, 485511168) 0 0 00000080:00200000:2.0:1788201707.021889:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.021893:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118532, 118532] 00020000:00200000:2.0:1788201707.021895:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485507072 chunk: [485507072, 485511168) 485511168, 4096 00020000:00200000:2.0:1788201707.021896:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.021903:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118532, 118532] 00000008:00010000:2.0:1788201707.021907:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.021908:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.021913:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021919:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485507072 485511167) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021924:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021929:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.021933:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.021937:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485507072, 485511168) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.021940:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118532 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.021944:0:17463:0:(rw26.c:898:ll_write_end()) pos 485507072, len 4096, copied 4096 00000080:00200000:2.0:1788201707.021946:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000347c6ff8[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118532] 00000080:00200000:2.0:1788201707.021948:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c1aad247 fffffc0001029 2:0 ffff93c0c11ae380 118532 lru 00000080:00200000:2.0:1788201707.021952:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000dc22d4fd 118532: 1< 2 - - > 2< 485507072 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70117 - -> 00000080:00200000:2.0:1788201707.021953:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000347c6ff8 00000080:00200000:2.0:1788201707.021954:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.021955:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.021963:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.021979:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.021982:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.021984:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.021993:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.021995:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.021997:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.022000:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022005:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.022022:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.022023:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.022026:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.022028:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118532, 118532] 00000080:00200000:2.0:1788201707.022035:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.022037:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.022056:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.022058:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.022068:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485511168, count: 4096 00000080:00200000:2.0:1788201707.022074:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485511168, bytes: 4096 00000080:00200000:2.0:1788201707.022076:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.022078:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.022079:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485511168, 485515264) 0 0 00000080:00200000:2.0:1788201707.022080:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.022084:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118533, 118533] 00020000:00200000:2.0:1788201707.022086:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485511168 chunk: [485511168, 485515264) 485515264, 4096 00020000:00200000:2.0:1788201707.022088:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.022095:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118533, 118533] 00000008:00010000:2.0:1788201707.022098:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.022099:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.022104:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022110:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485511168 485515263) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022116:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022120:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022124:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.022129:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485511168, 485515264) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.022132:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118533 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.022135:0:17463:0:(rw26.c:898:ll_write_end()) pos 485511168, len 4096, copied 4096 00000080:00200000:2.0:1788201707.022138:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f9978697[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118533] 00000080:00200000:2.0:1788201707.022140:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d718c046 fffffc0001029 2:0 ffff93c0c11ae8c0 118533 lru 00000080:00200000:2.0:1788201707.022143:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000460790e6 118533: 1< 2 - - > 2< 485511168 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70118 - -> 00000080:00200000:2.0:1788201707.022145:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f9978697 00000080:00200000:2.0:1788201707.022145:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.022147:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.022153:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.022160:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.022162:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.022163:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.022169:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.022171:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.022173:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.022175:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022180:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.022185:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.022186:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.022190:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.022192:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118533, 118533] 00000080:00200000:2.0:1788201707.022202:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.022205:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.022232:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.022236:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.022249:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485515264, count: 4096 00000080:00200000:2.0:1788201707.022255:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485515264, bytes: 4096 00000080:00200000:2.0:1788201707.022257:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.022259:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.022261:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485515264, 485519360) 0 0 00000080:00200000:2.0:1788201707.022263:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.022266:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118534, 118534] 00020000:00200000:2.0:1788201707.022268:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485515264 chunk: [485515264, 485519360) 485519360, 4096 00020000:00200000:2.0:1788201707.022270:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.022277:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118534, 118534] 00000008:00010000:2.0:1788201707.022280:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.022281:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.022286:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022292:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485515264 485519359) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022298:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022302:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022306:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.022311:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485515264, 485519360) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.022314:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118534 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.022318:0:17463:0:(rw26.c:898:ll_write_end()) pos 485515264, len 4096, copied 4096 00000080:00200000:2.0:1788201707.022320:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000042b0e6e0[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118534] 00000080:00200000:2.0:1788201707.022322:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000379a66ec fffffc0001029 2:0 ffff93c0c11aeee0 118534 lru 00000080:00200000:2.0:1788201707.022326:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000424e6dc2 118534: 1< 2 - - > 2< 485515264 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70119 - -> 00000080:00200000:2.0:1788201707.022329:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000042b0e6e0 00000080:00200000:2.0:1788201707.022329:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.022331:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.022336:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.022344:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.022346:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.022347:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.022353:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.022355:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.022357:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.022359:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022363:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.022368:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.022369:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.022372:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.022374:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118534, 118534] 00000080:00200000:2.0:1788201707.022380:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.022382:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.022400:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.022402:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.022412:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485519360, count: 4096 00000080:00200000:2.0:1788201707.022417:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485519360, bytes: 4096 00000080:00200000:2.0:1788201707.022419:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.022421:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.022422:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485519360, 485523456) 0 0 00000080:00200000:2.0:1788201707.022424:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.022427:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118535, 118535] 00020000:00200000:2.0:1788201707.022429:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485519360 chunk: [485519360, 485523456) 485523456, 4096 00020000:00200000:2.0:1788201707.022430:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.022437:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118535, 118535] 00000008:00010000:2.0:1788201707.022440:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.022441:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.022446:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022452:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485519360 485523455) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022457:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022462:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022465:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.022469:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485519360, 485523456) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.022472:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118535 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.022476:0:17463:0:(rw26.c:898:ll_write_end()) pos 485519360, len 4096, copied 4096 00000080:00200000:2.0:1788201707.022478:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000051bcc0e0[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118535] 00000080:00200000:2.0:1788201707.022480:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d9cd59d0 fffffc0001029 2:0 ffff93c0c11ae7e0 118535 lru 00000080:00200000:2.0:1788201707.022484:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000affc0830 118535: 1< 2 - - > 2< 485519360 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70120 - -> 00000080:00200000:2.0:1788201707.022485:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000051bcc0e0 00000080:00200000:2.0:1788201707.022486:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.022487:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.022492:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.022499:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.022501:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.022502:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.022508:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.022510:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.022511:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.022513:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022518:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.022523:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.022524:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.022526:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.022528:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118535, 118535] 00000080:00200000:2.0:1788201707.022534:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.022537:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.022554:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.022556:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.022566:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485523456, count: 4096 00000080:00200000:2.0:1788201707.022571:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485523456, bytes: 4096 00000080:00200000:2.0:1788201707.022573:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.022575:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.022576:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485523456, 485527552) 0 0 00000080:00200000:2.0:1788201707.022578:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.022581:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118536, 118536] 00020000:00200000:2.0:1788201707.022583:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485523456 chunk: [485523456, 485527552) 485527552, 4096 00020000:00200000:2.0:1788201707.022584:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.022591:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118536, 118536] 00000008:00010000:2.0:1788201707.022594:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.022595:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.022600:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022605:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485523456 485527551) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022611:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022616:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022619:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.022623:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485523456, 485527552) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.022626:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118536 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.022630:0:17463:0:(rw26.c:898:ll_write_end()) pos 485523456, len 4096, copied 4096 00000080:00200000:2.0:1788201707.022632:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004d269de5[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118536] 00000080:00200000:2.0:1788201707.022634:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000689b62d1 fffffc0001029 2:0 ffff93c0c11ae2a0 118536 lru 00000080:00200000:2.0:1788201707.022638:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000051d6abd9 118536: 1< 2 - - > 2< 485523456 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70121 - -> 00000080:00200000:2.0:1788201707.022639:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004d269de5 00000080:00200000:2.0:1788201707.022640:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.022641:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.022646:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.022654:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.022656:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.022657:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.022663:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.022665:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.022666:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.022668:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022673:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.022678:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.022679:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.022682:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.022684:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118536, 118536] 00000080:00200000:2.0:1788201707.022690:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.022692:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.022709:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.022711:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.022721:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485527552, count: 4096 00000080:00200000:2.0:1788201707.022726:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485527552, bytes: 4096 00000080:00200000:2.0:1788201707.022728:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.022730:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.022731:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485527552, 485531648) 0 0 00000080:00200000:2.0:1788201707.022733:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.022736:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118537, 118537] 00020000:00200000:2.0:1788201707.022738:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485527552 chunk: [485527552, 485531648) 485531648, 4096 00020000:00200000:2.0:1788201707.022739:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.022746:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118537, 118537] 00000008:00010000:2.0:1788201707.022749:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.022750:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.022755:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022761:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485527552 485531647) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022766:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022770:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022774:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.022778:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485527552, 485531648) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.022781:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118537 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.022785:0:17463:0:(rw26.c:898:ll_write_end()) pos 485527552, len 4096, copied 4096 00000080:00200000:2.0:1788201707.022789:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b6868583[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118537] 00000080:00200000:2.0:1788201707.022791:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008478f237 fffffc0001029 2:0 ffff93c0c11aee00 118537 lru 00000080:00200000:2.0:1788201707.022797:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a489c2a7 118537: 1< 2 - - > 2< 485527552 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70122 - -> 00000080:00200000:2.0:1788201707.022799:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b6868583 00000080:00200000:2.0:1788201707.022800:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.022802:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.022810:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.022822:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.022825:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.022827:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.022849:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.022851:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.022853:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.022856:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022862:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.022867:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.022869:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.022872:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.022874:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118537, 118537] 00000080:00200000:2.0:1788201707.022881:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.022883:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.022901:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.022903:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.022913:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485531648, count: 4096 00000080:00200000:2.0:1788201707.022919:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485531648, bytes: 4096 00000080:00200000:2.0:1788201707.022921:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.022923:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.022924:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485531648, 485535744) 0 0 00000080:00200000:2.0:1788201707.022925:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.022928:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118538, 118538] 00020000:00200000:2.0:1788201707.022930:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485531648 chunk: [485531648, 485535744) 485535744, 4096 00020000:00200000:2.0:1788201707.022932:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.022948:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118538, 118538] 00000008:00010000:2.0:1788201707.022954:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.022955:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.022961:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022966:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485531648 485535743) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022972:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022976:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.022980:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.022988:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485531648, 485535744) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.022991:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118538 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.022995:0:17463:0:(rw26.c:898:ll_write_end()) pos 485531648, len 4096, copied 4096 00000080:00200000:2.0:1788201707.022997:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005257a743[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118538] 00000080:00200000:2.0:1788201707.022999:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000974e2682 fffffc0001029 2:0 ffff93c0c11ae1c0 118538 lru 00000080:00200000:2.0:1788201707.023003:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000099d2e044 118538: 1< 2 - - > 2< 485531648 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70123 - -> 00000080:00200000:2.0:1788201707.023004:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005257a743 00000080:00200000:2.0:1788201707.023005:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.023006:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.023024:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.023033:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.023036:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.023037:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.023044:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.023045:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.023047:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.023050:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023054:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.023059:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.023061:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.023064:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.023066:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118538, 118538] 00000080:00200000:2.0:1788201707.023072:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.023074:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.023092:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.023094:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.023104:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485535744, count: 4096 00000080:00200000:2.0:1788201707.023110:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485535744, bytes: 4096 00000080:00200000:2.0:1788201707.023112:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.023114:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.023115:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485535744, 485539840) 0 0 00000080:00200000:2.0:1788201707.023116:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.023120:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118539, 118539] 00020000:00200000:2.0:1788201707.023121:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485535744 chunk: [485535744, 485539840) 485539840, 4096 00020000:00200000:2.0:1788201707.023123:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.023129:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118539, 118539] 00000008:00010000:2.0:1788201707.023132:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.023134:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.023139:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023145:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485535744 485539839) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023150:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023154:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023162:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.023166:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485535744, 485539840) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.023169:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118539 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.023173:0:17463:0:(rw26.c:898:ll_write_end()) pos 485535744, len 4096, copied 4096 00000080:00200000:2.0:1788201707.023175:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000032f6f3f4[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118539] 00000080:00200000:2.0:1788201707.023176:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c587c826 fffffc0001029 2:0 ffff93c0c11ae620 118539 lru 00000080:00200000:2.0:1788201707.023180:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000086dad578 118539: 1< 2 - - > 2< 485535744 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70124 - -> 00000080:00200000:2.0:1788201707.023181:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000032f6f3f4 00000080:00200000:2.0:1788201707.023182:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.023183:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.023189:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.023200:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.023203:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.023206:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.023215:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.023218:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.023220:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.023224:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023233:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.023245:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.023246:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.023249:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.023252:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118539, 118539] 00000080:00200000:2.0:1788201707.023259:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.023261:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.023280:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.023282:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.023292:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485539840, count: 4096 00000080:00200000:2.0:1788201707.023298:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485539840, bytes: 4096 00000080:00200000:2.0:1788201707.023299:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.023302:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.023303:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485539840, 485543936) 0 0 00000080:00200000:2.0:1788201707.023304:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.023307:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118540, 118540] 00020000:00200000:2.0:1788201707.023309:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485539840 chunk: [485539840, 485543936) 485543936, 4096 00020000:00200000:2.0:1788201707.023311:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.023318:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118540, 118540] 00000008:00010000:2.0:1788201707.023321:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.023322:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.023327:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023333:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485539840 485543935) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023339:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023347:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023351:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.023355:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485539840, 485543936) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.023358:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118540 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.023362:0:17463:0:(rw26.c:898:ll_write_end()) pos 485539840, len 4096, copied 4096 00000080:00200000:2.0:1788201707.023364:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f06b52b8[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118540] 00000080:00200000:2.0:1788201707.023365:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000672f2bc5 fffffc0001029 2:0 ffff93c0c11aed20 118540 lru 00000080:00200000:2.0:1788201707.023369:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000092e1b37c 118540: 1< 2 - - > 2< 485539840 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70125 - -> 00000080:00200000:2.0:1788201707.023370:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f06b52b8 00000080:00200000:2.0:1788201707.023371:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.023372:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.023377:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.023385:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.023387:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.023388:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.023395:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.023396:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.023398:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.023400:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023408:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.023413:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.023414:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.023417:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.023418:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118540, 118540] 00000080:00200000:2.0:1788201707.023425:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.023427:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.023444:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.023446:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.023456:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485543936, count: 4096 00000080:00200000:2.0:1788201707.023462:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485543936, bytes: 4096 00000080:00200000:2.0:1788201707.023464:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.023466:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.023467:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485543936, 485548032) 0 0 00000080:00200000:2.0:1788201707.023468:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.023471:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118541, 118541] 00020000:00200000:2.0:1788201707.023473:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485543936 chunk: [485543936, 485548032) 485548032, 4096 00020000:00200000:2.0:1788201707.023475:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.023481:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118541, 118541] 00000008:00010000:2.0:1788201707.023484:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.023485:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.023490:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023496:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485543936 485548031) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023505:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023509:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023513:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.023517:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485543936, 485548032) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.023520:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118541 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.023523:0:17463:0:(rw26.c:898:ll_write_end()) pos 485543936, len 4096, copied 4096 00000080:00200000:2.0:1788201707.023525:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000741aac5e[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118541] 00000080:00200000:2.0:1788201707.023527:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000806288e9 fffffc0001029 2:0 ffff93c0c11ae9a0 118541 lru 00000080:00200000:2.0:1788201707.023531:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ab8bdb3f 118541: 1< 2 - - > 2< 485543936 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70126 - -> 00000080:00200000:2.0:1788201707.023532:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000741aac5e 00000080:00200000:2.0:1788201707.023533:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.023534:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.023539:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.023546:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.023548:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.023549:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.023555:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.023557:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.023559:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.023561:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023566:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.023571:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.023572:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.023575:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.023577:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118541, 118541] 00000080:00200000:2.0:1788201707.023583:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.023585:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.023602:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.023604:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.023614:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485548032, count: 4096 00000080:00200000:2.0:1788201707.023620:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485548032, bytes: 4096 00000080:00200000:2.0:1788201707.023622:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.023624:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.023625:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485548032, 485552128) 0 0 00000080:00200000:2.0:1788201707.023626:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.023630:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118542, 118542] 00020000:00200000:2.0:1788201707.023632:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485548032 chunk: [485548032, 485552128) 485552128, 4096 00020000:00200000:2.0:1788201707.023633:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.023639:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118542, 118542] 00000008:00010000:2.0:1788201707.023642:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.023644:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.023648:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023658:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485548032 485552127) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023663:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023667:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023671:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.023675:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485548032, 485552128) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.023678:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118542 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.023681:0:17463:0:(rw26.c:898:ll_write_end()) pos 485548032, len 4096, copied 4096 00000080:00200000:2.0:1788201707.023683:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000be25468b[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118542] 00000080:00200000:2.0:1788201707.023685:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004be6ec32 fffffc0001029 2:0 ffff93c0c11aeb60 118542 lru 00000080:00200000:2.0:1788201707.023688:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d5cc8a35 118542: 1< 2 - - > 2< 485548032 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70127 - -> 00000080:00200000:2.0:1788201707.023690:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000be25468b 00000080:00200000:2.0:1788201707.023690:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.023691:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.023696:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.023704:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.023705:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.023707:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.023712:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.023714:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.023716:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.023718:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023723:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.023728:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.023729:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.023732:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.023734:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118542, 118542] 00000080:00200000:2.0:1788201707.023744:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.023747:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.023776:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.023779:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.023794:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485552128, count: 4096 00000080:00200000:2.0:1788201707.023804:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485552128, bytes: 4096 00000080:00200000:2.0:1788201707.023806:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.023810:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.023812:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485552128, 485556224) 0 0 00000080:00200000:2.0:1788201707.023814:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.023819:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118543, 118543] 00020000:00200000:2.0:1788201707.023821:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485552128 chunk: [485552128, 485556224) 485556224, 4096 00020000:00200000:2.0:1788201707.023824:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.023853:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118543, 118543] 00000008:00010000:2.0:1788201707.023859:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.023861:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.023871:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023879:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485552128 485556223) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023888:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023895:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023902:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.023910:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485552128, 485556224) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.023915:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118543 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.023920:0:17463:0:(rw26.c:898:ll_write_end()) pos 485552128, len 4096, copied 4096 00000080:00200000:2.0:1788201707.023924:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000627c1056[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118543] 00000080:00200000:2.0:1788201707.023927:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a0cea8cd fffffc0001029 2:0 ffff93c0c11afa80 118543 lru 00000080:00200000:2.0:1788201707.023933:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a471763d 118543: 1< 2 - - > 2< 485552128 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70128 - -> 00000080:00200000:2.0:1788201707.023935:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000627c1056 00000080:00200000:2.0:1788201707.023937:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.023939:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.023946:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.023955:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.023957:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.023958:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.023966:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.023967:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.023969:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.023972:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.023976:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.023982:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.023983:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.023985:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.023987:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118543, 118543] 00000080:00200000:2.0:1788201707.023994:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.023996:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.024029:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.024032:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.024042:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485556224, count: 4096 00000080:00200000:2.0:1788201707.024049:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485556224, bytes: 4096 00000080:00200000:2.0:1788201707.024051:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.024053:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.024054:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485556224, 485560320) 0 0 00000080:00200000:2.0:1788201707.024056:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.024059:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118544, 118544] 00020000:00200000:2.0:1788201707.024061:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485556224 chunk: [485556224, 485560320) 485560320, 4096 00020000:00200000:2.0:1788201707.024063:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.024069:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118544, 118544] 00000008:00010000:2.0:1788201707.024073:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.024074:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.024080:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024086:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485556224 485560319) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024091:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024095:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024099:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.024104:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485556224, 485560320) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.024107:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118544 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.024110:0:17463:0:(rw26.c:898:ll_write_end()) pos 485556224, len 4096, copied 4096 00000080:00200000:2.0:1788201707.024112:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005bcb51de[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118544] 00000080:00200000:2.0:1788201707.024114:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001431f461 fffffc0001029 2:0 ffff93c0c11af380 118544 lru 00000080:00200000:2.0:1788201707.024118:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000df3bc166 118544: 1< 2 - - > 2< 485556224 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70129 - -> 00000080:00200000:2.0:1788201707.024119:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005bcb51de 00000080:00200000:2.0:1788201707.024120:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.024121:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.024127:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.024138:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.024139:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.024141:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.024147:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.024149:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.024150:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.024152:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024157:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.024162:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.024164:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.024166:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.024168:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118544, 118544] 00000080:00200000:2.0:1788201707.024174:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.024176:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.024198:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.024201:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.024215:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485560320, count: 4096 00000080:00200000:2.0:1788201707.024226:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485560320, bytes: 4096 00000080:00200000:2.0:1788201707.024229:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.024233:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.024234:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485560320, 485564416) 0 0 00000080:00200000:2.0:1788201707.024236:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.024242:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118545, 118545] 00020000:00200000:2.0:1788201707.024244:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485560320 chunk: [485560320, 485564416) 485564416, 4096 00020000:00200000:2.0:1788201707.024246:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.024254:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118545, 118545] 00000008:00010000:2.0:1788201707.024258:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.024259:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.024265:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024271:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485560320 485564415) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024276:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024281:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024284:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.024289:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485560320, 485564416) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.024292:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118545 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.024296:0:17463:0:(rw26.c:898:ll_write_end()) pos 485560320, len 4096, copied 4096 00000080:00200000:2.0:1788201707.024299:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fe03f287[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118545] 00000080:00200000:2.0:1788201707.024300:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000959d89aa fffffc0001029 2:0 ffff93c0c11af8c0 118545 lru 00000080:00200000:2.0:1788201707.024304:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000228ad3c6 118545: 1< 2 - - > 2< 485560320 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70130 - -> 00000080:00200000:2.0:1788201707.024306:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fe03f287 00000080:00200000:2.0:1788201707.024306:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.024308:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.024314:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.024322:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.024323:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.024325:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.024331:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.024332:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.024334:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.024336:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024341:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.024346:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.024347:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.024350:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.024352:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118545, 118545] 00000080:00200000:2.0:1788201707.024358:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.024360:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.024377:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.024380:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.024389:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485564416, count: 4096 00000080:00200000:2.0:1788201707.024395:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485564416, bytes: 4096 00000080:00200000:2.0:1788201707.024397:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.024399:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.024400:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485564416, 485568512) 0 0 00000080:00200000:2.0:1788201707.024402:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.024405:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118546, 118546] 00020000:00200000:2.0:1788201707.024407:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485564416 chunk: [485564416, 485568512) 485568512, 4096 00020000:00200000:2.0:1788201707.024408:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.024415:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118546, 118546] 00000008:00010000:2.0:1788201707.024418:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.024419:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.024425:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024434:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485564416 485568511) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024442:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024450:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024457:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.024466:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485564416, 485568512) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.024471:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118546 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.024476:0:17463:0:(rw26.c:898:ll_write_end()) pos 485564416, len 4096, copied 4096 00000080:00200000:2.0:1788201707.024479:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000007a0addf[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118546] 00000080:00200000:2.0:1788201707.024481:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000fc58832 fffffc0001029 2:0 ffff93c0c11afee0 118546 lru 00000080:00200000:2.0:1788201707.024485:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000fb63e957 118546: 1< 2 - - > 2< 485564416 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70131 - -> 00000080:00200000:2.0:1788201707.024487:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000007a0addf 00000080:00200000:2.0:1788201707.024488:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.024489:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.024495:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.024503:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.024505:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.024507:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.024513:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.024514:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.024516:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.024519:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024523:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.024528:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.024529:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.024532:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.024534:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118546, 118546] 00000080:00200000:2.0:1788201707.024541:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.024542:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.024560:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.024562:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.024572:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485568512, count: 4096 00000080:00200000:2.0:1788201707.024578:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485568512, bytes: 4096 00000080:00200000:2.0:1788201707.024580:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.024582:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.024583:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485568512, 485572608) 0 0 00000080:00200000:2.0:1788201707.024585:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.024588:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118547, 118547] 00020000:00200000:2.0:1788201707.024590:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485568512 chunk: [485568512, 485572608) 485572608, 4096 00020000:00200000:2.0:1788201707.024591:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.024598:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118547, 118547] 00000008:00010000:2.0:1788201707.024601:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.024602:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.024607:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024613:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485568512 485572607) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024619:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024623:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024627:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.024631:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485568512, 485572608) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.024634:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118547 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.024637:0:17463:0:(rw26.c:898:ll_write_end()) pos 485568512, len 4096, copied 4096 00000080:00200000:2.0:1788201707.024639:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000de0864ab[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118547] 00000080:00200000:2.0:1788201707.024641:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000669a0c9f fffffc0001029 2:0 ffff93c0c11af7e0 118547 lru 00000080:00200000:2.0:1788201707.024645:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000017990d10 118547: 1< 2 - - > 2< 485568512 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70132 - -> 00000080:00200000:2.0:1788201707.024646:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000de0864ab 00000080:00200000:2.0:1788201707.024647:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.024648:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.024653:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.024660:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.024662:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.024663:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.024669:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.024671:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.024672:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.024675:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024679:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.024684:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.024685:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.024688:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.024690:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118547, 118547] 00000080:00200000:2.0:1788201707.024696:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.024698:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.024715:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.024717:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.024727:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485572608, count: 4096 00000080:00200000:2.0:1788201707.024733:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485572608, bytes: 4096 00000080:00200000:2.0:1788201707.024734:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.024736:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.024737:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485572608, 485576704) 0 0 00000080:00200000:2.0:1788201707.024739:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.024742:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118548, 118548] 00020000:00200000:2.0:1788201707.024744:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485572608 chunk: [485572608, 485576704) 485576704, 4096 00020000:00200000:2.0:1788201707.024745:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.024752:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118548, 118548] 00000008:00010000:2.0:1788201707.024755:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.024756:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.024761:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024766:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485572608 485576703) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024772:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024776:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024780:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.024785:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485572608, 485576704) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.024787:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118548 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.024791:0:17463:0:(rw26.c:898:ll_write_end()) pos 485572608, len 4096, copied 4096 00000080:00200000:2.0:1788201707.024794:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000e104abe[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118548] 00000080:00200000:2.0:1788201707.024796:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000049cd73a fffffc0001029 2:0 ffff93c0c11af2a0 118548 lru 00000080:00200000:2.0:1788201707.024799:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ff68a0ae 118548: 1< 2 - - > 2< 485572608 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70133 - -> 00000080:00200000:2.0:1788201707.024801:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000e104abe 00000080:00200000:2.0:1788201707.024801:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.024803:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.024808:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.024815:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.024817:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.024818:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.024824:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.024826:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.024828:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.024850:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024855:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.024861:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.024862:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.024866:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.024868:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118548, 118548] 00000080:00200000:2.0:1788201707.024878:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.024880:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.024898:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.024901:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.024910:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485576704, count: 4096 00000080:00200000:2.0:1788201707.024916:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485576704, bytes: 4096 00000080:00200000:2.0:1788201707.024918:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.024920:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.024921:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485576704, 485580800) 0 0 00000080:00200000:2.0:1788201707.024923:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.024926:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118549, 118549] 00020000:00200000:2.0:1788201707.024927:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485576704 chunk: [485576704, 485580800) 485580800, 4096 00020000:00200000:2.0:1788201707.024929:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.024935:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118549, 118549] 00000008:00010000:2.0:1788201707.024939:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.024940:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.024945:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024951:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485576704 485580799) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024956:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024960:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.024964:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.024968:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485576704, 485580800) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.024971:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118549 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.024975:0:17463:0:(rw26.c:898:ll_write_end()) pos 485576704, len 4096, copied 4096 00000080:00200000:2.0:1788201707.024977:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005611ad08[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118549] 00000080:00200000:2.0:1788201707.024979:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cb8a431e fffffc0001029 2:0 ffff93c0c11afe00 118549 lru 00000080:00200000:2.0:1788201707.024983:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006a53629b 118549: 1< 2 - - > 2< 485576704 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70134 - -> 00000080:00200000:2.0:1788201707.024984:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005611ad08 00000080:00200000:2.0:1788201707.024985:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.024986:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.024991:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.024999:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.025001:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.025002:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.025021:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.025023:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.025025:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.025028:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025033:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.025038:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.025039:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.025042:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.025044:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118549, 118549] 00000080:00200000:2.0:1788201707.025051:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.025053:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.025071:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.025073:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.025084:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485580800, count: 4096 00000080:00200000:2.0:1788201707.025089:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485580800, bytes: 4096 00000080:00200000:2.0:1788201707.025091:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.025093:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.025094:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485580800, 485584896) 0 0 00000080:00200000:2.0:1788201707.025096:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.025099:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118550, 118550] 00020000:00200000:2.0:1788201707.025101:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485580800 chunk: [485580800, 485584896) 485584896, 4096 00020000:00200000:2.0:1788201707.025102:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.025109:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118550, 118550] 00000008:00010000:2.0:1788201707.025112:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.025113:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.025118:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025124:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485580800 485584895) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025129:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025134:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025137:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.025145:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485580800, 485584896) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.025148:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118550 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.025151:0:17463:0:(rw26.c:898:ll_write_end()) pos 485580800, len 4096, copied 4096 00000080:00200000:2.0:1788201707.025154:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000029cbc7b3[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118550] 00000080:00200000:2.0:1788201707.025156:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000aec54dd6 fffffc0001029 2:0 ffff93c0c11af1c0 118550 lru 00000080:00200000:2.0:1788201707.025160:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ed309a22 118550: 1< 2 - - > 2< 485580800 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70135 - -> 00000080:00200000:2.0:1788201707.025161:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000029cbc7b3 00000080:00200000:2.0:1788201707.025162:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.025163:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.025168:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.025176:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.025177:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.025179:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.025185:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.025187:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.025189:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.025193:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025201:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.025209:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.025212:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.025217:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.025220:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118550, 118550] 00000080:00200000:2.0:1788201707.025230:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.025233:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.025254:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.025256:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.025266:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485584896, count: 4096 00000080:00200000:2.0:1788201707.025272:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485584896, bytes: 4096 00000080:00200000:2.0:1788201707.025274:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.025276:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.025277:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485584896, 485588992) 0 0 00000080:00200000:2.0:1788201707.025279:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.025282:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118551, 118551] 00020000:00200000:2.0:1788201707.025284:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485584896 chunk: [485584896, 485588992) 485588992, 4096 00020000:00200000:2.0:1788201707.025285:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.025292:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118551, 118551] 00000008:00010000:2.0:1788201707.025295:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.025297:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.025302:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025308:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485584896 485588991) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025313:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025318:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025325:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.025329:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485584896, 485588992) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.025332:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118551 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.025336:0:17463:0:(rw26.c:898:ll_write_end()) pos 485584896, len 4096, copied 4096 00000080:00200000:2.0:1788201707.025338:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bf4dd61a[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118551] 00000080:00200000:2.0:1788201707.025340:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008d42f837 fffffc0001029 2:0 ffff93c0c11af620 118551 lru 00000080:00200000:2.0:1788201707.025343:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001b21c763 118551: 1< 2 - - > 2< 485584896 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70136 - -> 00000080:00200000:2.0:1788201707.025345:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bf4dd61a 00000080:00200000:2.0:1788201707.025346:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.025347:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.025352:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.025360:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.025362:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.025363:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.025369:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.025371:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.025373:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.025375:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025380:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.025388:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.025389:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.025392:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.025394:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118551, 118551] 00000080:00200000:2.0:1788201707.025400:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.025402:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.025419:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.025422:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.025436:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485588992, count: 4096 00000080:00200000:2.0:1788201707.025447:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485588992, bytes: 4096 00000080:00200000:2.0:1788201707.025449:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.025453:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.025454:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485588992, 485593088) 0 0 00000080:00200000:2.0:1788201707.025457:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.025462:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118552, 118552] 00020000:00200000:2.0:1788201707.025465:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485588992 chunk: [485588992, 485593088) 485593088, 4096 00020000:00200000:2.0:1788201707.025467:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.025477:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118552, 118552] 00000008:00010000:2.0:1788201707.025481:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.025482:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.025488:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025494:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485588992 485593087) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025499:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025507:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025512:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.025516:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485588992, 485593088) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.025519:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118552 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.025522:0:17463:0:(rw26.c:898:ll_write_end()) pos 485588992, len 4096, copied 4096 00000080:00200000:2.0:1788201707.025525:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006d910b36[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118552] 00000080:00200000:2.0:1788201707.025526:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e47048ce fffffc0001029 2:0 ffff93c0c11afd20 118552 lru 00000080:00200000:2.0:1788201707.025530:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000395d4a9e 118552: 1< 2 - - > 2< 485588992 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70137 - -> 00000080:00200000:2.0:1788201707.025531:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006d910b36 00000080:00200000:2.0:1788201707.025531:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.025532:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.025538:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.025546:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.025548:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.025549:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.025555:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.025557:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.025558:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.025561:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025569:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.025574:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.025575:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.025577:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.025579:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118552, 118552] 00000080:00200000:2.0:1788201707.025586:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.025588:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.025605:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.025607:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.025617:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485593088, count: 4096 00000080:00200000:2.0:1788201707.025623:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485593088, bytes: 4096 00000080:00200000:2.0:1788201707.025624:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.025627:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.025628:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485593088, 485597184) 0 0 00000080:00200000:2.0:1788201707.025629:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.025632:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118553, 118553] 00020000:00200000:2.0:1788201707.025634:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485593088 chunk: [485593088, 485597184) 485597184, 4096 00020000:00200000:2.0:1788201707.025635:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.025642:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118553, 118553] 00000008:00010000:2.0:1788201707.025645:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.025646:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.025651:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025657:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485593088 485597183) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025665:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025670:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025673:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.025678:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485593088, 485597184) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.025681:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118553 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.025684:0:17463:0:(rw26.c:898:ll_write_end()) pos 485593088, len 4096, copied 4096 00000080:00200000:2.0:1788201707.025687:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000000619143[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118553] 00000080:00200000:2.0:1788201707.025689:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002af55c45 fffffc0001029 2:0 ffff93c0c4416d20 118553 lru 00000080:00200000:2.0:1788201707.025692:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d8113c08 118553: 1< 2 - - > 2< 485593088 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70138 - -> 00000080:00200000:2.0:1788201707.025694:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000000619143 00000080:00200000:2.0:1788201707.025694:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.025696:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.025701:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.025708:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.025710:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.025711:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.025717:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.025718:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.025720:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.025722:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025727:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.025732:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.025733:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.025736:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.025738:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118553, 118553] 00000080:00200000:2.0:1788201707.025744:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.025746:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.025763:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.025765:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.025775:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485597184, count: 4096 00000080:00200000:2.0:1788201707.025781:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485597184, bytes: 4096 00000080:00200000:2.0:1788201707.025782:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.025784:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.025785:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485597184, 485601280) 0 0 00000080:00200000:2.0:1788201707.025787:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.025790:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118554, 118554] 00020000:00200000:2.0:1788201707.025792:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485597184 chunk: [485597184, 485601280) 485601280, 4096 00020000:00200000:2.0:1788201707.025793:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.025799:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118554, 118554] 00000008:00010000:2.0:1788201707.025803:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.025804:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.025809:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025818:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485597184 485601279) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025823:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025827:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025844:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.025849:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485597184, 485601280) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.025852:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118554 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.025856:0:17463:0:(rw26.c:898:ll_write_end()) pos 485597184, len 4096, copied 4096 00000080:00200000:2.0:1788201707.025859:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000017fed67c[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118554] 00000080:00200000:2.0:1788201707.025860:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b0c6af45 fffffc0001029 2:0 ffff93c0c44169a0 118554 lru 00000080:00200000:2.0:1788201707.025864:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002829bca3 118554: 1< 2 - - > 2< 485597184 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70139 - -> 00000080:00200000:2.0:1788201707.025865:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000017fed67c 00000080:00200000:2.0:1788201707.025866:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.025867:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.025873:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.025880:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.025882:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.025883:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.025889:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.025891:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.025893:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.025895:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025900:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.025905:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.025906:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.025909:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.025910:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118554, 118554] 00000080:00200000:2.0:1788201707.025917:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.025919:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.025936:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.025938:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.025948:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485601280, count: 4096 00000080:00200000:2.0:1788201707.025954:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485601280, bytes: 4096 00000080:00200000:2.0:1788201707.025955:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.025958:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.025959:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485601280, 485605376) 0 0 00000080:00200000:2.0:1788201707.025960:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.025963:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118555, 118555] 00020000:00200000:2.0:1788201707.025965:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485601280 chunk: [485601280, 485605376) 485605376, 4096 00020000:00200000:2.0:1788201707.025967:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.025973:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118555, 118555] 00000008:00010000:2.0:1788201707.025976:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.025978:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.025983:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025988:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485601280 485605375) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025994:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.025998:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026002:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.026007:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485601280, 485605376) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.026022:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118555 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.026026:0:17463:0:(rw26.c:898:ll_write_end()) pos 485601280, len 4096, copied 4096 00000080:00200000:2.0:1788201707.026029:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000009062b04[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118555] 00000080:00200000:2.0:1788201707.026031:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000130de94f fffffc0001029 2:0 ffff93c0c4416b60 118555 lru 00000080:00200000:2.0:1788201707.026034:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ed410dd0 118555: 1< 2 - - > 2< 485601280 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70140 - -> 00000080:00200000:2.0:1788201707.026036:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000009062b04 00000080:00200000:2.0:1788201707.026036:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.026038:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.026043:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.026051:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.026053:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.026054:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.026060:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.026062:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.026064:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.026066:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026071:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.026076:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.026077:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.026080:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.026082:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118555, 118555] 00000080:00200000:2.0:1788201707.026088:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.026090:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.026107:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.026110:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.026120:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485605376, count: 4096 00000080:00200000:2.0:1788201707.026125:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485605376, bytes: 4096 00000080:00200000:2.0:1788201707.026127:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.026129:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.026130:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485605376, 485609472) 0 0 00000080:00200000:2.0:1788201707.026132:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.026135:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118556, 118556] 00020000:00200000:2.0:1788201707.026137:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485605376 chunk: [485605376, 485609472) 485609472, 4096 00020000:00200000:2.0:1788201707.026138:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.026145:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118556, 118556] 00000008:00010000:2.0:1788201707.026148:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.026149:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.026154:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026160:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485605376 485609471) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026165:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026170:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026173:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.026178:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485605376, 485609472) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.026181:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118556 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.026185:0:17463:0:(rw26.c:898:ll_write_end()) pos 485605376, len 4096, copied 4096 00000080:00200000:2.0:1788201707.026188:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f7eccf8d[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118556] 00000080:00200000:2.0:1788201707.026191:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cc2a7fe2 fffffc0001029 2:0 ffff93c0c4416c40 118556 lru 00000080:00200000:2.0:1788201707.026196:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001e5e85b6 118556: 1< 2 - - > 2< 485605376 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70141 - -> 00000080:00200000:2.0:1788201707.026199:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f7eccf8d 00000080:00200000:2.0:1788201707.026200:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.026202:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.026210:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.026226:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.026229:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.026232:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.026240:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.026243:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.026246:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.026249:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026256:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.026261:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.026263:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.026266:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.026268:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118556, 118556] 00000080:00200000:2.0:1788201707.026275:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.026277:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.026295:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.026297:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.026307:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485609472, count: 4096 00000080:00200000:2.0:1788201707.026313:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485609472, bytes: 4096 00000080:00200000:2.0:1788201707.026315:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.026317:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.026318:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485609472, 485613568) 0 0 00000080:00200000:2.0:1788201707.026320:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.026323:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118557, 118557] 00020000:00200000:2.0:1788201707.026325:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485609472 chunk: [485609472, 485613568) 485613568, 4096 00020000:00200000:2.0:1788201707.026327:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.026333:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118557, 118557] 00000008:00010000:2.0:1788201707.026336:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.026338:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.026343:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026348:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485609472 485613567) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026354:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026359:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026363:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.026367:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485609472, 485613568) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.026369:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118557 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.026373:0:17463:0:(rw26.c:898:ll_write_end()) pos 485609472, len 4096, copied 4096 00000080:00200000:2.0:1788201707.026375:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002c739402[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118557] 00000080:00200000:2.0:1788201707.026377:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000091196c6e fffffc0001029 2:0 ffff93c0c4416540 118557 lru 00000080:00200000:2.0:1788201707.026380:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001fe885e5 118557: 1< 2 - - > 2< 485609472 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70142 - -> 00000080:00200000:2.0:1788201707.026382:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002c739402 00000080:00200000:2.0:1788201707.026383:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.026384:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.026389:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.026397:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.026399:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.026400:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.026406:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.026408:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.026410:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.026412:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026417:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.026423:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.026425:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.026429:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.026431:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118557, 118557] 00000080:00200000:2.0:1788201707.026441:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.026444:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.026471:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.026475:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.026486:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485613568, count: 4096 00000080:00200000:2.0:1788201707.026492:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485613568, bytes: 4096 00000080:00200000:2.0:1788201707.026494:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.026496:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.026497:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485613568, 485617664) 0 0 00000080:00200000:2.0:1788201707.026499:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.026502:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118558, 118558] 00020000:00200000:2.0:1788201707.026504:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485613568 chunk: [485613568, 485617664) 485617664, 4096 00020000:00200000:2.0:1788201707.026506:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.026513:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118558, 118558] 00000008:00010000:2.0:1788201707.026516:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.026517:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.026523:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026529:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485613568 485617663) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026534:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026538:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026542:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.026547:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485613568, 485617664) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.026550:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118558 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.026553:0:17463:0:(rw26.c:898:ll_write_end()) pos 485613568, len 4096, copied 4096 00000080:00200000:2.0:1788201707.026556:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000080dd27b3[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118558] 00000080:00200000:2.0:1788201707.026558:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000094c27b6e fffffc0001029 2:0 ffff93c0c4416460 118558 lru 00000080:00200000:2.0:1788201707.026561:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ee586d4a 118558: 1< 2 - - > 2< 485613568 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70143 - -> 00000080:00200000:2.0:1788201707.026564:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000080dd27b3 00000080:00200000:2.0:1788201707.026564:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.026566:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.026571:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.026579:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.026581:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.026582:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.026588:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.026590:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.026592:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.026594:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026598:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.026603:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.026604:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.026607:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.026609:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118558, 118558] 00000080:00200000:2.0:1788201707.026615:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.026617:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.026634:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.026636:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.026646:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485617664, count: 4096 00000080:00200000:2.0:1788201707.026652:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485617664, bytes: 4096 00000080:00200000:2.0:1788201707.026653:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.026656:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.026657:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485617664, 485621760) 0 0 00000080:00200000:2.0:1788201707.026658:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.026661:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118559, 118559] 00020000:00200000:2.0:1788201707.026663:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485617664 chunk: [485617664, 485621760) 485621760, 4096 00020000:00200000:2.0:1788201707.026665:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.026671:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118559, 118559] 00000008:00010000:2.0:1788201707.026674:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.026675:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.026680:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026686:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485617664 485621759) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026691:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026696:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026700:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.026704:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485617664, 485621760) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.026706:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118559 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.026710:0:17463:0:(rw26.c:898:ll_write_end()) pos 485617664, len 4096, copied 4096 00000080:00200000:2.0:1788201707.026712:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000caf4bae4[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118559] 00000080:00200000:2.0:1788201707.026714:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003d7c6870 fffffc0001029 2:0 ffff93c0c44160e0 118559 lru 00000080:00200000:2.0:1788201707.026717:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000fee451cb 118559: 1< 2 - - > 2< 485617664 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70144 - -> 00000080:00200000:2.0:1788201707.026719:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000caf4bae4 00000080:00200000:2.0:1788201707.026719:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.026721:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.026726:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.026733:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.026735:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.026736:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.026742:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.026743:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.026745:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.026747:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026751:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.026756:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.026757:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.026760:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.026762:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118559, 118559] 00000080:00200000:2.0:1788201707.026768:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.026770:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.026787:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.026789:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.026799:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485621760, count: 4096 00000080:00200000:2.0:1788201707.026805:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485621760, bytes: 4096 00000080:00200000:2.0:1788201707.026806:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.026808:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.026809:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485621760, 485625856) 0 0 00000080:00200000:2.0:1788201707.026811:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.026814:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118560, 118560] 00020000:00200000:2.0:1788201707.026816:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485621760 chunk: [485621760, 485625856) 485625856, 4096 00020000:00200000:2.0:1788201707.026817:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.026824:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118560, 118560] 00000008:00010000:2.0:1788201707.026827:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.026828:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.026846:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026852:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485621760 485625855) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026857:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026862:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026866:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.026870:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485621760, 485625856) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.026873:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118560 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.026877:0:17463:0:(rw26.c:898:ll_write_end()) pos 485621760, len 4096, copied 4096 00000080:00200000:2.0:1788201707.026880:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b19a6a03[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118560] 00000080:00200000:2.0:1788201707.026881:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009f5579e3 fffffc0001029 2:0 ffff93c0c4416700 118560 lru 00000080:00200000:2.0:1788201707.026885:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000091cdce72 118560: 1< 2 - - > 2< 485621760 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70145 - -> 00000080:00200000:2.0:1788201707.026886:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b19a6a03 00000080:00200000:2.0:1788201707.026887:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.026888:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.026894:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.026901:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.026903:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.026904:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.026911:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.026912:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.026914:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.026916:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.026921:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.026926:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.026927:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.026929:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.026931:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118560, 118560] 00000080:00200000:2.0:1788201707.026937:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.026939:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.026957:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.026959:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.026969:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485625856, count: 4096 00000080:00200000:2.0:1788201707.026974:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485625856, bytes: 4096 00000080:00200000:2.0:1788201707.026976:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.026978:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.026979:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485625856, 485629952) 0 0 00000080:00200000:2.0:1788201707.026981:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.026984:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118561, 118561] 00020000:00200000:2.0:1788201707.026986:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485625856 chunk: [485625856, 485629952) 485629952, 4096 00020000:00200000:2.0:1788201707.026987:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.026993:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118561, 118561] 00000008:00010000:2.0:1788201707.026996:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.026998:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.027003:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027021:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485625856 485629951) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027028:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027033:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027036:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.027041:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485625856, 485629952) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.027045:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118561 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.027049:0:17463:0:(rw26.c:898:ll_write_end()) pos 485625856, len 4096, copied 4096 00000080:00200000:2.0:1788201707.027051:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006b60e2fa[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118561] 00000080:00200000:2.0:1788201707.027053:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000033eb3363 fffffc0001029 2:0 ffff93c0c4416000 118561 lru 00000080:00200000:2.0:1788201707.027057:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000090e7b9f9 118561: 1< 2 - - > 2< 485625856 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70146 - -> 00000080:00200000:2.0:1788201707.027058:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006b60e2fa 00000080:00200000:2.0:1788201707.027059:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.027062:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.027068:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.027076:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.027077:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.027079:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.027085:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.027086:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.027088:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.027090:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027095:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.027100:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.027101:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.027104:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.027106:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118561, 118561] 00000080:00200000:2.0:1788201707.027112:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.027114:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.027132:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.027134:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.027144:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485629952, count: 4096 00000080:00200000:2.0:1788201707.027150:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485629952, bytes: 4096 00000080:00200000:2.0:1788201707.027152:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.027154:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.027155:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485629952, 485634048) 0 0 00000080:00200000:2.0:1788201707.027157:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.027160:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118562, 118562] 00020000:00200000:2.0:1788201707.027162:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485629952 chunk: [485629952, 485634048) 485634048, 4096 00020000:00200000:2.0:1788201707.027163:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.027170:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118562, 118562] 00000008:00010000:2.0:1788201707.027173:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.027174:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.027179:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027185:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485629952 485634047) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027193:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027200:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027207:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.027221:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485629952, 485634048) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.027226:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118562 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.027231:0:17463:0:(rw26.c:898:ll_write_end()) pos 485629952, len 4096, copied 4096 00000080:00200000:2.0:1788201707.027235:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000dbad0144[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118562] 00000080:00200000:2.0:1788201707.027237:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000954cc139 fffffc0001029 2:0 ffff93c0c4416a80 118562 lru 00000080:00200000:2.0:1788201707.027241:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006c5931ac 118562: 1< 2 - - > 2< 485629952 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70147 - -> 00000080:00200000:2.0:1788201707.027243:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000dbad0144 00000080:00200000:2.0:1788201707.027243:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.027245:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.027251:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.027259:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.027261:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.027262:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.027268:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.027270:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.027272:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.027275:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027279:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.027284:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.027286:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.027289:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.027291:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118562, 118562] 00000080:00200000:2.0:1788201707.027297:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.027299:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.027317:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.027319:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.027329:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485634048, count: 4096 00000080:00200000:2.0:1788201707.027335:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485634048, bytes: 4096 00000080:00200000:2.0:1788201707.027337:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.027339:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.027340:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485634048, 485638144) 0 0 00000080:00200000:2.0:1788201707.027342:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.027345:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118563, 118563] 00020000:00200000:2.0:1788201707.027347:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485634048 chunk: [485634048, 485638144) 485638144, 4096 00020000:00200000:2.0:1788201707.027348:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.027355:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118563, 118563] 00000008:00010000:2.0:1788201707.027358:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.027359:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.027364:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027370:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485634048 485638143) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027375:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027380:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027387:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.027392:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485634048, 485638144) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.027394:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118563 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.027398:0:17463:0:(rw26.c:898:ll_write_end()) pos 485634048, len 4096, copied 4096 00000080:00200000:2.0:1788201707.027400:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f227760e[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118563] 00000080:00200000:2.0:1788201707.027402:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f6ad87a6 fffffc0001029 2:0 ffff93c0c4416380 118563 lru 00000080:00200000:2.0:1788201707.027405:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ffa2d2c4 118563: 1< 2 - - > 2< 485634048 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70148 - -> 00000080:00200000:2.0:1788201707.027407:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f227760e 00000080:00200000:2.0:1788201707.027407:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.027408:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.027414:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.027421:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.027424:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.027426:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.027435:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.027437:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.027440:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.027443:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027452:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.027465:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.027467:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.027472:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.027474:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118563, 118563] 00000080:00200000:2.0:1788201707.027481:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.027483:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.027501:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.027504:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.027516:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485638144, count: 4096 00000080:00200000:2.0:1788201707.027526:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485638144, bytes: 4096 00000080:00200000:2.0:1788201707.027529:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.027533:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.027534:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485638144, 485642240) 0 0 00000080:00200000:2.0:1788201707.027537:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.027541:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118564, 118564] 00020000:00200000:2.0:1788201707.027544:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485638144 chunk: [485638144, 485642240) 485642240, 4096 00020000:00200000:2.0:1788201707.027547:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.027558:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118564, 118564] 00000008:00010000:2.0:1788201707.027563:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.027564:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.027570:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027576:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485638144 485642239) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027582:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027590:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027594:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.027598:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485638144, 485642240) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.027601:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118564 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.027604:0:17463:0:(rw26.c:898:ll_write_end()) pos 485638144, len 4096, copied 4096 00000080:00200000:2.0:1788201707.027607:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bf4349a5[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118564] 00000080:00200000:2.0:1788201707.027609:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000072cf1af9 fffffc0001029 2:0 ffff93c0c44168c0 118564 lru 00000080:00200000:2.0:1788201707.027612:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000687af45c 118564: 1< 2 - - > 2< 485638144 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70149 - -> 00000080:00200000:2.0:1788201707.027613:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bf4349a5 00000080:00200000:2.0:1788201707.027614:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.027615:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.027620:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.027628:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.027630:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.027632:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.027638:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.027640:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.027641:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.027643:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027651:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.027656:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.027657:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.027660:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.027662:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118564, 118564] 00000080:00200000:2.0:1788201707.027668:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.027670:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.027688:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.027690:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.027700:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485642240, count: 4096 00000080:00200000:2.0:1788201707.027706:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485642240, bytes: 4096 00000080:00200000:2.0:1788201707.027708:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.027710:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.027711:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485642240, 485646336) 0 0 00000080:00200000:2.0:1788201707.027712:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.027716:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118565, 118565] 00020000:00200000:2.0:1788201707.027717:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485642240 chunk: [485642240, 485646336) 485646336, 4096 00020000:00200000:2.0:1788201707.027719:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.027725:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118565, 118565] 00000008:00010000:2.0:1788201707.027728:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.027730:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.027734:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027740:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485642240 485646335) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027749:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027753:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027757:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.027762:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485642240, 485646336) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.027765:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118565 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.027768:0:17463:0:(rw26.c:898:ll_write_end()) pos 485642240, len 4096, copied 4096 00000080:00200000:2.0:1788201707.027771:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e51bc595[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118565] 00000080:00200000:2.0:1788201707.027772:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000064a803fb fffffc0001029 2:0 ffff93c0c4416ee0 118565 lru 00000080:00200000:2.0:1788201707.027776:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000010cc8207 118565: 1< 2 - - > 2< 485642240 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70150 - -> 00000080:00200000:2.0:1788201707.027777:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e51bc595 00000080:00200000:2.0:1788201707.027778:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.027779:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.027784:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.027792:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.027793:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.027795:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.027801:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.027802:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.027804:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.027806:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027811:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.027816:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.027818:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.027820:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.027822:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118565, 118565] 00000080:00200000:2.0:1788201707.027846:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.027848:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.027867:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.027869:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.027879:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485646336, count: 4096 00000080:00200000:2.0:1788201707.027885:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485646336, bytes: 4096 00000080:00200000:2.0:1788201707.027887:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.027889:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.027891:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485646336, 485650432) 0 0 00000080:00200000:2.0:1788201707.027892:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.027896:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118566, 118566] 00020000:00200000:2.0:1788201707.027898:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485646336 chunk: [485646336, 485650432) 485650432, 4096 00020000:00200000:2.0:1788201707.027900:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.027906:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118566, 118566] 00000008:00010000:2.0:1788201707.027909:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.027911:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.027916:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027925:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485646336 485650431) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027930:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027935:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027939:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.027943:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485646336, 485650432) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.027946:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118566 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.027949:0:17463:0:(rw26.c:898:ll_write_end()) pos 485646336, len 4096, copied 4096 00000080:00200000:2.0:1788201707.027952:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fdd4aaf1[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118566] 00000080:00200000:2.0:1788201707.027953:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006ae168b4 fffffc0001029 2:0 ffff93c0c4417540 118566 lru 00000080:00200000:2.0:1788201707.027957:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000002cb8a8e 118566: 1< 2 - - > 2< 485646336 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70151 - -> 00000080:00200000:2.0:1788201707.027958:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fdd4aaf1 00000080:00200000:2.0:1788201707.027959:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.027960:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.027965:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.027972:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.027974:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.027976:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.027982:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.027984:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.027985:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.027987:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.027992:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.027997:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.027999:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.028001:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.028003:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118566, 118566] 00000080:00200000:2.0:1788201707.028022:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.028025:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.028044:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.028046:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.028056:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485650432, count: 4096 00000080:00200000:2.0:1788201707.028062:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485650432, bytes: 4096 00000080:00200000:2.0:1788201707.028064:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.028066:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.028067:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485650432, 485654528) 0 0 00000080:00200000:2.0:1788201707.028068:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.028072:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118567, 118567] 00020000:00200000:2.0:1788201707.028074:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485650432 chunk: [485650432, 485654528) 485654528, 4096 00020000:00200000:2.0:1788201707.028075:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.028082:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118567, 118567] 00000008:00010000:2.0:1788201707.028085:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.028086:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.028092:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028097:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485650432 485654527) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028103:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028107:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028111:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.028116:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485650432, 485654528) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.028119:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118567 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.028122:0:17463:0:(rw26.c:898:ll_write_end()) pos 485650432, len 4096, copied 4096 00000080:00200000:2.0:1788201707.028125:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a9690bf0[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118567] 00000080:00200000:2.0:1788201707.028126:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000033c29063 fffffc0001029 2:0 ffff93c0c4417460 118567 lru 00000080:00200000:2.0:1788201707.028130:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005d98f25f 118567: 1< 2 - - > 2< 485650432 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70152 - -> 00000080:00200000:2.0:1788201707.028131:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a9690bf0 00000080:00200000:2.0:1788201707.028132:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.028133:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.028139:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.028146:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.028148:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.028149:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.028155:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.028157:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.028159:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.028161:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028166:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.028171:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.028172:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.028174:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.028176:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118567, 118567] 00000080:00200000:2.0:1788201707.028182:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.028184:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.028209:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.028214:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.028229:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485654528, count: 4096 00000080:00200000:2.0:1788201707.028239:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485654528, bytes: 4096 00000080:00200000:2.0:1788201707.028241:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.028244:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.028245:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485654528, 485658624) 0 0 00000080:00200000:2.0:1788201707.028246:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.028250:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118568, 118568] 00020000:00200000:2.0:1788201707.028252:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485654528 chunk: [485654528, 485658624) 485658624, 4096 00020000:00200000:2.0:1788201707.028253:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.028261:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118568, 118568] 00000008:00010000:2.0:1788201707.028264:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.028266:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.028271:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028277:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485654528 485658623) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028283:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028287:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028291:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.028296:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485654528, 485658624) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.028299:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118568 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.028303:0:17463:0:(rw26.c:898:ll_write_end()) pos 485654528, len 4096, copied 4096 00000080:00200000:2.0:1788201707.028306:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007432d21c[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118568] 00000080:00200000:2.0:1788201707.028307:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001b733c9a fffffc0001029 2:0 ffff93c0c44170e0 118568 lru 00000080:00200000:2.0:1788201707.028311:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004fa8e13e 118568: 1< 2 - - > 2< 485654528 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70153 - -> 00000080:00200000:2.0:1788201707.028312:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007432d21c 00000080:00200000:2.0:1788201707.028313:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.028314:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.028320:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.028331:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.028332:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.028334:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.028340:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.028341:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.028343:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.028345:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028350:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.028356:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.028357:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.028359:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.028361:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118568, 118568] 00000080:00200000:2.0:1788201707.028367:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.028369:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.028387:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.028389:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.028399:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485658624, count: 4096 00000080:00200000:2.0:1788201707.028405:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485658624, bytes: 4096 00000080:00200000:2.0:1788201707.028406:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.028408:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.028410:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485658624, 485662720) 0 0 00000080:00200000:2.0:1788201707.028411:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.028414:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118569, 118569] 00020000:00200000:2.0:1788201707.028416:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485658624 chunk: [485658624, 485662720) 485662720, 4096 00020000:00200000:2.0:1788201707.028418:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.028427:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118569, 118569] 00000008:00010000:2.0:1788201707.028432:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.028434:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.028441:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028450:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485658624 485662719) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028459:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028467:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028474:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.028480:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485658624, 485662720) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.028484:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118569 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.028487:0:17463:0:(rw26.c:898:ll_write_end()) pos 485658624, len 4096, copied 4096 00000080:00200000:2.0:1788201707.028490:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009af3fa63[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118569] 00000080:00200000:2.0:1788201707.028492:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000001e34088 fffffc0001029 2:0 ffff93c0c4417700 118569 lru 00000080:00200000:2.0:1788201707.028496:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000fd9ac054 118569: 1< 2 - - > 2< 485658624 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70154 - -> 00000080:00200000:2.0:1788201707.028497:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009af3fa63 00000080:00200000:2.0:1788201707.028498:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.028500:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.028506:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.028517:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.028520:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.028522:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.028531:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.028534:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.028537:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.028541:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028549:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.028557:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.028558:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.028562:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.028564:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118569, 118569] 00000080:00200000:2.0:1788201707.028571:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.028573:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.028591:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.028594:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.028604:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485662720, count: 4096 00000080:00200000:2.0:1788201707.028609:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485662720, bytes: 4096 00000080:00200000:2.0:1788201707.028611:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.028613:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.028614:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485662720, 485666816) 0 0 00000080:00200000:2.0:1788201707.028616:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.028619:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118570, 118570] 00020000:00200000:2.0:1788201707.028621:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485662720 chunk: [485662720, 485666816) 485666816, 4096 00020000:00200000:2.0:1788201707.028623:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.028629:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118570, 118570] 00000008:00010000:2.0:1788201707.028632:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.028634:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.028639:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028645:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485662720 485666815) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028651:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028655:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028659:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.028663:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485662720, 485666816) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.028667:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118570 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.028670:0:17463:0:(rw26.c:898:ll_write_end()) pos 485662720, len 4096, copied 4096 00000080:00200000:2.0:1788201707.028672:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000069ca92d7[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118570] 00000080:00200000:2.0:1788201707.028674:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d28858f9 fffffc0001029 2:0 ffff93c0c4417000 118570 lru 00000080:00200000:2.0:1788201707.028678:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000df8fc94 118570: 1< 2 - - > 2< 485662720 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70155 - -> 00000080:00200000:2.0:1788201707.028680:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000069ca92d7 00000080:00200000:2.0:1788201707.028681:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.028682:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.028688:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.028696:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.028697:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.028699:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.028705:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.028706:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.028708:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.028710:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028715:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.028720:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.028721:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.028724:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.028726:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118570, 118570] 00000080:00200000:2.0:1788201707.028732:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.028734:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.028751:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.028753:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.028763:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485666816, count: 4096 00000080:00200000:2.0:1788201707.028769:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485666816, bytes: 4096 00000080:00200000:2.0:1788201707.028771:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.028773:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.028774:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485666816, 485670912) 0 0 00000080:00200000:2.0:1788201707.028775:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.028778:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118571, 118571] 00020000:00200000:2.0:1788201707.028780:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485666816 chunk: [485666816, 485670912) 485670912, 4096 00020000:00200000:2.0:1788201707.028781:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.028788:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118571, 118571] 00000008:00010000:2.0:1788201707.028791:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.028792:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.028797:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028803:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485666816 485670911) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028808:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028813:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028816:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.028820:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485666816, 485670912) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.028823:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118571 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.028826:0:17463:0:(rw26.c:898:ll_write_end()) pos 485666816, len 4096, copied 4096 00000080:00200000:2.0:1788201707.028842:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b35c06b2[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118571] 00000080:00200000:2.0:1788201707.028844:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a0614967 fffffc0001029 2:0 ffff93c0c4417a80 118571 lru 00000080:00200000:2.0:1788201707.028848:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002de11695 118571: 1< 2 - - > 2< 485666816 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70156 - -> 00000080:00200000:2.0:1788201707.028849:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b35c06b2 00000080:00200000:2.0:1788201707.028850:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.028851:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.028857:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.028864:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.028866:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.028867:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.028873:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.028875:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.028876:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.028879:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028884:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.028888:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.028890:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.028892:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.028894:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118571, 118571] 00000080:00200000:2.0:1788201707.028900:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.028902:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.028919:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.028921:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.028931:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485670912, count: 4096 00000080:00200000:2.0:1788201707.028937:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485670912, bytes: 4096 00000080:00200000:2.0:1788201707.028939:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.028941:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.028942:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485670912, 485675008) 0 0 00000080:00200000:2.0:1788201707.028944:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.028947:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118572, 118572] 00020000:00200000:2.0:1788201707.028948:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485670912 chunk: [485670912, 485675008) 485675008, 4096 00020000:00200000:2.0:1788201707.028950:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.028956:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118572, 118572] 00000008:00010000:2.0:1788201707.028959:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.028961:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.028965:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028971:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485670912 485675007) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028977:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028981:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.028985:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.028989:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485670912, 485675008) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.028992:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118572 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.028995:0:17463:0:(rw26.c:898:ll_write_end()) pos 485670912, len 4096, copied 4096 00000080:00200000:2.0:1788201707.028998:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009bade0b2[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118572] 00000080:00200000:2.0:1788201707.029000:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c0d28def fffffc0001029 2:0 ffff93c0c4417380 118572 lru 00000080:00200000:2.0:1788201707.029003:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001eb3da96 118572: 1< 2 - - > 2< 485670912 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70157 - -> 00000080:00200000:2.0:1788201707.029005:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009bade0b2 00000080:00200000:2.0:1788201707.029005:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.029007:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.029024:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.029032:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.029034:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.029036:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.029042:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.029044:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.029046:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.029048:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029053:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.029058:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.029059:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.029062:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.029064:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118572, 118572] 00000080:00200000:2.0:1788201707.029070:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.029072:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.029090:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.029092:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.029102:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485675008, count: 4096 00000080:00200000:2.0:1788201707.029108:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485675008, bytes: 4096 00000080:00200000:2.0:1788201707.029110:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.029112:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.029113:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485675008, 485679104) 0 0 00000080:00200000:2.0:1788201707.029115:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.029118:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118573, 118573] 00020000:00200000:2.0:1788201707.029120:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485675008 chunk: [485675008, 485679104) 485679104, 4096 00020000:00200000:2.0:1788201707.029121:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.029128:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118573, 118573] 00000008:00010000:2.0:1788201707.029131:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.029132:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.029137:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029143:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485675008 485679103) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029148:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029153:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029157:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.029161:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485675008, 485679104) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.029164:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118573 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.029168:0:17463:0:(rw26.c:898:ll_write_end()) pos 485675008, len 4096, copied 4096 00000080:00200000:2.0:1788201707.029170:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008d23bd72[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118573] 00000080:00200000:2.0:1788201707.029172:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005bc3659c fffffc0001029 2:0 ffff93c0c44178c0 118573 lru 00000080:00200000:2.0:1788201707.029175:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000dbf651c8 118573: 1< 2 - - > 2< 485675008 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70158 - -> 00000080:00200000:2.0:1788201707.029177:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008d23bd72 00000080:00200000:2.0:1788201707.029178:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.029179:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.029184:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.029195:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.029198:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.029200:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.029208:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.029211:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.029213:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.029217:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029226:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.029234:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.029237:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.029241:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.029243:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118573, 118573] 00000080:00200000:2.0:1788201707.029250:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.029252:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.029270:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.029273:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.029286:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485679104, count: 4096 00000080:00200000:2.0:1788201707.029296:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485679104, bytes: 4096 00000080:00200000:2.0:1788201707.029298:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.029302:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.029304:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485679104, 485683200) 0 0 00000080:00200000:2.0:1788201707.029307:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.029311:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118574, 118574] 00020000:00200000:2.0:1788201707.029314:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485679104 chunk: [485679104, 485683200) 485683200, 4096 00020000:00200000:2.0:1788201707.029316:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.029327:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118574, 118574] 00000008:00010000:2.0:1788201707.029330:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.029332:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.029337:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029343:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485679104 485683199) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029348:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029353:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029357:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.029365:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485679104, 485683200) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.029368:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118574 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.029372:0:17463:0:(rw26.c:898:ll_write_end()) pos 485679104, len 4096, copied 4096 00000080:00200000:2.0:1788201707.029374:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009d172d0c[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118574] 00000080:00200000:2.0:1788201707.029376:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007221282d fffffc0001029 2:0 ffff93c0c4417ee0 118574 lru 00000080:00200000:2.0:1788201707.029380:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000059ef09a7 118574: 1< 2 - - > 2< 485679104 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70159 - -> 00000080:00200000:2.0:1788201707.029381:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009d172d0c 00000080:00200000:2.0:1788201707.029382:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.029383:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.029389:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.029397:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.029398:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.029400:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.029406:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.029407:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.029409:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.029411:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029416:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.029423:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.029425:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.029429:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.029432:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118574, 118574] 00000080:00200000:2.0:1788201707.029441:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.029445:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.029472:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.029475:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.029486:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485683200, count: 4096 00000080:00200000:2.0:1788201707.029493:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485683200, bytes: 4096 00000080:00200000:2.0:1788201707.029495:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.029497:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.029498:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485683200, 485687296) 0 0 00000080:00200000:2.0:1788201707.029500:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.029503:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118575, 118575] 00020000:00200000:2.0:1788201707.029505:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485683200 chunk: [485683200, 485687296) 485687296, 4096 00020000:00200000:2.0:1788201707.029507:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.029517:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118575, 118575] 00000008:00010000:2.0:1788201707.029522:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.029524:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.029532:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029541:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485683200 485687295) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029550:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029558:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029566:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.029571:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485683200, 485687296) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.029575:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118575 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.029578:0:17463:0:(rw26.c:898:ll_write_end()) pos 485683200, len 4096, copied 4096 00000080:00200000:2.0:1788201707.029581:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004fcdd3ae[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118575] 00000080:00200000:2.0:1788201707.029582:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000adf56b07 fffffc0001029 2:0 ffff93c0c44177e0 118575 lru 00000080:00200000:2.0:1788201707.029586:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e7d5985c 118575: 1< 2 - - > 2< 485683200 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70160 - -> 00000080:00200000:2.0:1788201707.029588:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004fcdd3ae 00000080:00200000:2.0:1788201707.029589:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.029590:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.029596:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.029604:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.029606:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.029607:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.029614:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.029616:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.029617:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.029619:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029624:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.029633:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.029634:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.029637:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.029639:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118575, 118575] 00000080:00200000:2.0:1788201707.029646:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.029648:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.029665:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.029668:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.029678:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485687296, count: 4096 00000080:00200000:2.0:1788201707.029683:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485687296, bytes: 4096 00000080:00200000:2.0:1788201707.029685:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.029687:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.029689:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485687296, 485691392) 0 0 00000080:00200000:2.0:1788201707.029690:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.029693:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118576, 118576] 00020000:00200000:2.0:1788201707.029695:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485687296 chunk: [485687296, 485691392) 485691392, 4096 00020000:00200000:2.0:1788201707.029696:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.029703:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118576, 118576] 00000008:00010000:2.0:1788201707.029706:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.029708:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.029713:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029718:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485687296 485691391) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029724:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029732:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029736:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.029740:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485687296, 485691392) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.029743:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118576 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.029746:0:17463:0:(rw26.c:898:ll_write_end()) pos 485687296, len 4096, copied 4096 00000080:00200000:2.0:1788201707.029748:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b6995682[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118576] 00000080:00200000:2.0:1788201707.029750:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000707a7da fffffc0001029 2:0 ffff93c0c13f7e00 118576 lru 00000080:00200000:2.0:1788201707.029753:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005cd9070c 118576: 1< 2 - - > 2< 485687296 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70161 - -> 00000080:00200000:2.0:1788201707.029754:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b6995682 00000080:00200000:2.0:1788201707.029755:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.029756:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.029761:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.029769:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.029771:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.029772:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.029778:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.029779:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.029781:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.029783:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029791:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.029796:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.029797:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.029800:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.029801:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118576, 118576] 00000080:00200000:2.0:1788201707.029807:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.029809:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.029826:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.029840:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.029851:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485691392, count: 4096 00000080:00200000:2.0:1788201707.029858:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485691392, bytes: 4096 00000080:00200000:2.0:1788201707.029860:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.029862:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.029864:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485691392, 485695488) 0 0 00000080:00200000:2.0:1788201707.029865:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.029869:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118577, 118577] 00020000:00200000:2.0:1788201707.029871:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485691392 chunk: [485691392, 485695488) 485695488, 4096 00020000:00200000:2.0:1788201707.029873:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.029879:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118577, 118577] 00000008:00010000:2.0:1788201707.029883:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.029884:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.029889:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029894:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485691392 485695487) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029903:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029908:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029912:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.029916:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485691392, 485695488) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.029919:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118577 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.029922:0:17463:0:(rw26.c:898:ll_write_end()) pos 485691392, len 4096, copied 4096 00000080:00200000:2.0:1788201707.029924:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008a66312a[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118577] 00000080:00200000:2.0:1788201707.029926:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000086430ee3 fffffc0001029 2:0 ffff93c0c13f71c0 118577 lru 00000080:00200000:2.0:1788201707.029929:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a5a3629d 118577: 1< 2 - - > 2< 485691392 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70162 - -> 00000080:00200000:2.0:1788201707.029930:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008a66312a 00000080:00200000:2.0:1788201707.029931:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.029932:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.029938:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.029946:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.029947:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.029948:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.029955:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.029956:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.029958:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.029961:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.029966:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.029971:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.029972:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.029975:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.029976:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118577, 118577] 00000080:00200000:2.0:1788201707.029983:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.029985:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.030002:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.030004:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.030027:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485695488, count: 4096 00000080:00200000:2.0:1788201707.030034:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485695488, bytes: 4096 00000080:00200000:2.0:1788201707.030036:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.030038:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.030039:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485695488, 485699584) 0 0 00000080:00200000:2.0:1788201707.030040:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.030044:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118578, 118578] 00020000:00200000:2.0:1788201707.030046:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485695488 chunk: [485695488, 485699584) 485699584, 4096 00020000:00200000:2.0:1788201707.030047:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.030054:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118578, 118578] 00000008:00010000:2.0:1788201707.030057:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.030058:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.030063:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030073:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485695488 485699583) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030078:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030082:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030086:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.030091:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485695488, 485699584) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.030094:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118578 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.030098:0:17463:0:(rw26.c:898:ll_write_end()) pos 485695488, len 4096, copied 4096 00000080:00200000:2.0:1788201707.030100:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000720ac79e[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118578] 00000080:00200000:2.0:1788201707.030102:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bb035484 fffffc0001029 2:0 ffff93c0c13f7620 118578 lru 00000080:00200000:2.0:1788201707.030106:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a00520c5 118578: 1< 2 - - > 2< 485695488 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70163 - -> 00000080:00200000:2.0:1788201707.030107:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000720ac79e 00000080:00200000:2.0:1788201707.030108:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.030109:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.030115:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.030122:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.030124:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.030126:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.030132:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.030134:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.030135:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.030137:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030142:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.030147:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.030148:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.030151:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.030153:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118578, 118578] 00000080:00200000:2.0:1788201707.030159:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.030161:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.030178:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.030181:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.030192:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485699584, count: 4096 00000080:00200000:2.0:1788201707.030201:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485699584, bytes: 4096 00000080:00200000:2.0:1788201707.030204:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.030208:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.030209:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485699584, 485703680) 0 0 00000080:00200000:2.0:1788201707.030211:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.030216:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118579, 118579] 00020000:00200000:2.0:1788201707.030219:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485699584 chunk: [485699584, 485703680) 485703680, 4096 00020000:00200000:2.0:1788201707.030222:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.030233:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118579, 118579] 00000008:00010000:2.0:1788201707.030238:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.030240:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.030246:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030252:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485699584 485703679) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030257:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030261:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030265:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.030270:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485699584, 485703680) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.030275:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118579 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.030280:0:17463:0:(rw26.c:898:ll_write_end()) pos 485699584, len 4096, copied 4096 00000080:00200000:2.0:1788201707.030284:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004d597bd7[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118579] 00000080:00200000:2.0:1788201707.030287:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000803e7ea4 fffffc0001029 2:0 ffff93c0c13f7d20 118579 lru 00000080:00200000:2.0:1788201707.030293:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000073375648 118579: 1< 2 - - > 2< 485699584 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70164 - -> 00000080:00200000:2.0:1788201707.030295:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004d597bd7 00000080:00200000:2.0:1788201707.030296:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.030298:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.030307:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.030319:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.030322:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.030324:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.030333:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.030336:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.030339:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.030343:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030349:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.030355:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.030357:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.030360:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.030362:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118579, 118579] 00000080:00200000:2.0:1788201707.030369:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.030371:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.030390:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.030392:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.030402:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485703680, count: 4096 00000080:00200000:2.0:1788201707.030408:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485703680, bytes: 4096 00000080:00200000:2.0:1788201707.030410:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.030412:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.030413:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485703680, 485707776) 0 0 00000080:00200000:2.0:1788201707.030415:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.030418:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118580, 118580] 00020000:00200000:2.0:1788201707.030421:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485703680 chunk: [485703680, 485707776) 485707776, 4096 00020000:00200000:2.0:1788201707.030423:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.030433:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118580, 118580] 00000008:00010000:2.0:1788201707.030438:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.030441:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.030448:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030457:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485703680 485707775) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030466:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030474:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030478:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.030484:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485703680, 485707776) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.030487:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118580 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.030491:0:17463:0:(rw26.c:898:ll_write_end()) pos 485703680, len 4096, copied 4096 00000080:00200000:2.0:1788201707.030493:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000996e952a[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118580] 00000080:00200000:2.0:1788201707.030495:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ff488522 fffffc0001029 2:0 ffff93c0c13f79a0 118580 lru 00000080:00200000:2.0:1788201707.030499:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f5d426bc 118580: 1< 2 - - > 2< 485703680 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70165 - -> 00000080:00200000:2.0:1788201707.030500:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000996e952a 00000080:00200000:2.0:1788201707.030501:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.030502:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.030509:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.030526:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.030529:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.030532:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.030540:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.030543:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.030546:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.030550:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030558:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.030563:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.030565:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.030568:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.030570:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118580, 118580] 00000080:00200000:2.0:1788201707.030577:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.030579:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.030598:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.030600:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.030610:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485707776, count: 4096 00000080:00200000:2.0:1788201707.030616:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485707776, bytes: 4096 00000080:00200000:2.0:1788201707.030618:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.030620:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.030621:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485707776, 485711872) 0 0 00000080:00200000:2.0:1788201707.030623:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.030626:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118581, 118581] 00020000:00200000:2.0:1788201707.030628:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485707776 chunk: [485707776, 485711872) 485711872, 4096 00020000:00200000:2.0:1788201707.030629:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.030636:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118581, 118581] 00000008:00010000:2.0:1788201707.030639:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.030641:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.030646:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030652:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485707776 485711871) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030657:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030661:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030665:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.030670:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485707776, 485711872) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.030673:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118581 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.030676:0:17463:0:(rw26.c:898:ll_write_end()) pos 485707776, len 4096, copied 4096 00000080:00200000:2.0:1788201707.030679:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000de200193[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118581] 00000080:00200000:2.0:1788201707.030681:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009f70b94f fffffc0001029 2:0 ffff93c0c13f7b60 118581 lru 00000080:00200000:2.0:1788201707.030684:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000888e96ac 118581: 1< 2 - - > 2< 485707776 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70166 - -> 00000080:00200000:2.0:1788201707.030686:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000de200193 00000080:00200000:2.0:1788201707.030686:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.030688:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.030693:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.030701:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.030703:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.030704:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.030710:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.030712:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.030713:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.030716:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030720:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.030725:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.030726:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.030729:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.030731:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118581, 118581] 00000080:00200000:2.0:1788201707.030737:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.030739:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.030756:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.030758:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.030767:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485711872, count: 4096 00000080:00200000:2.0:1788201707.030773:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485711872, bytes: 4096 00000080:00200000:2.0:1788201707.030775:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.030777:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.030778:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485711872, 485715968) 0 0 00000080:00200000:2.0:1788201707.030780:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.030783:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118582, 118582] 00020000:00200000:2.0:1788201707.030785:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485711872 chunk: [485711872, 485715968) 485715968, 4096 00020000:00200000:2.0:1788201707.030786:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.030793:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118582, 118582] 00000008:00010000:2.0:1788201707.030796:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.030797:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.030802:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030808:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485711872 485715967) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030813:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030817:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030821:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.030825:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485711872, 485715968) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.030828:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118582 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.030845:0:17463:0:(rw26.c:898:ll_write_end()) pos 485711872, len 4096, copied 4096 00000080:00200000:2.0:1788201707.030847:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000000cb3ab0[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118582] 00000080:00200000:2.0:1788201707.030849:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c02a7bd1 fffffc0001029 2:0 ffff93c0c13f7c40 118582 lru 00000080:00200000:2.0:1788201707.030853:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c07afc95 118582: 1< 2 - - > 2< 485711872 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70167 - -> 00000080:00200000:2.0:1788201707.030855:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000000cb3ab0 00000080:00200000:2.0:1788201707.030856:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.030857:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.030863:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.030870:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.030872:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.030873:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.030879:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.030881:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.030883:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.030885:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030890:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.030894:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.030896:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.030898:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.030900:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118582, 118582] 00000080:00200000:2.0:1788201707.030906:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.030908:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.030925:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.030927:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.030937:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485715968, count: 4096 00000080:00200000:2.0:1788201707.030942:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485715968, bytes: 4096 00000080:00200000:2.0:1788201707.030945:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.030947:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.030948:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485715968, 485720064) 0 0 00000080:00200000:2.0:1788201707.030949:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.030952:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118583, 118583] 00020000:00200000:2.0:1788201707.030954:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485715968 chunk: [485715968, 485720064) 485720064, 4096 00020000:00200000:2.0:1788201707.030956:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.030962:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118583, 118583] 00000008:00010000:2.0:1788201707.030965:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.030967:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.030972:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030977:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485715968 485720063) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030983:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030987:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.030991:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.030996:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485715968, 485720064) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.030999:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118583 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.031002:0:17463:0:(rw26.c:898:ll_write_end()) pos 485715968, len 4096, copied 4096 00000080:00200000:2.0:1788201707.031005:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000081b647d1[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118583] 00000080:00200000:2.0:1788201707.031007:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ceb68cf8 fffffc0001029 2:0 ffff93c0c13f7540 118583 lru 00000080:00200000:2.0:1788201707.031024:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000083146e1c 118583: 1< 2 - - > 2< 485715968 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70168 - -> 00000080:00200000:2.0:1788201707.031026:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000081b647d1 00000080:00200000:2.0:1788201707.031026:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.031028:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.031034:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.031042:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.031044:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.031046:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.031052:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.031054:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.031055:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.031058:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031063:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.031068:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.031069:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.031072:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.031074:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118583, 118583] 00000080:00200000:2.0:1788201707.031080:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.031082:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.031099:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.031102:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.031118:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485720064, count: 4096 00000080:00200000:2.0:1788201707.031128:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485720064, bytes: 4096 00000080:00200000:2.0:1788201707.031131:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.031135:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.031136:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485720064, 485724160) 0 0 00000080:00200000:2.0:1788201707.031139:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.031143:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118584, 118584] 00020000:00200000:2.0:1788201707.031146:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485720064 chunk: [485720064, 485724160) 485724160, 4096 00020000:00200000:2.0:1788201707.031148:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.031160:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118584, 118584] 00000008:00010000:2.0:1788201707.031165:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.031167:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.031174:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031180:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485720064 485724159) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031185:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031193:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031200:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.031208:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485720064, 485724160) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.031213:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118584 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.031219:0:17463:0:(rw26.c:898:ll_write_end()) pos 485720064, len 4096, copied 4096 00000080:00200000:2.0:1788201707.031223:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000037ff8b6a[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118584] 00000080:00200000:2.0:1788201707.031226:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009a7ce49b fffffc0001029 2:0 ffff93c0c13f7460 118584 lru 00000080:00200000:2.0:1788201707.031232:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000075ba26a0 118584: 1< 2 - - > 2< 485720064 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70169 - -> 00000080:00200000:2.0:1788201707.031234:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000037ff8b6a 00000080:00200000:2.0:1788201707.031235:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.031237:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.031244:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.031252:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.031253:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.031255:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.031262:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.031263:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.031265:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.031268:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031274:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.031282:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.031284:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.031289:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.031292:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118584, 118584] 00000080:00200000:2.0:1788201707.031302:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.031305:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.031333:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.031337:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.031351:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485724160, count: 4096 00000080:00200000:2.0:1788201707.031358:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485724160, bytes: 4096 00000080:00200000:2.0:1788201707.031359:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.031362:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.031363:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485724160, 485728256) 0 0 00000080:00200000:2.0:1788201707.031364:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.031368:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118585, 118585] 00020000:00200000:2.0:1788201707.031370:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485724160 chunk: [485724160, 485728256) 485728256, 4096 00020000:00200000:2.0:1788201707.031371:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.031378:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118585, 118585] 00000008:00010000:2.0:1788201707.031382:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.031383:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.031388:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031394:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485724160 485728255) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031400:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031404:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031409:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.031413:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485724160, 485728256) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.031417:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118585 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.031421:0:17463:0:(rw26.c:898:ll_write_end()) pos 485724160, len 4096, copied 4096 00000080:00200000:2.0:1788201707.031425:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000658d8575[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118585] 00000080:00200000:2.0:1788201707.031427:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000090becda4 fffffc0001029 2:0 ffff93c0c13f70e0 118585 lru 00000080:00200000:2.0:1788201707.031433:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000072a3ea00 118585: 1< 2 - - > 2< 485724160 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70170 - -> 00000080:00200000:2.0:1788201707.031435:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000658d8575 00000080:00200000:2.0:1788201707.031436:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.031437:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.031446:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.031458:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.031461:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.031463:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.031472:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.031474:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.031475:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.031478:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031483:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.031488:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.031489:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.031492:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.031494:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118585, 118585] 00000080:00200000:2.0:1788201707.031501:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.031503:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.031529:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.031533:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.031548:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485728256, count: 4096 00000080:00200000:2.0:1788201707.031558:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485728256, bytes: 4096 00000080:00200000:2.0:1788201707.031560:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.031562:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.031563:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485728256, 485732352) 0 0 00000080:00200000:2.0:1788201707.031565:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.031568:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118586, 118586] 00020000:00200000:2.0:1788201707.031570:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485728256 chunk: [485728256, 485732352) 485732352, 4096 00020000:00200000:2.0:1788201707.031572:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.031579:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118586, 118586] 00000008:00010000:2.0:1788201707.031584:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.031587:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.031594:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031603:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485728256 485732351) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031610:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031617:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031623:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.031636:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485728256, 485732352) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.031641:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118586 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.031646:0:17463:0:(rw26.c:898:ll_write_end()) pos 485728256, len 4096, copied 4096 00000080:00200000:2.0:1788201707.031649:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000954e9beb[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118586] 00000080:00200000:2.0:1788201707.031652:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004abcde9d fffffc0001029 2:0 ffff93c0c13f7700 118586 lru 00000080:00200000:2.0:1788201707.031657:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ebf3fb5c 118586: 1< 2 - - > 2< 485728256 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70171 - -> 00000080:00200000:2.0:1788201707.031659:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000954e9beb 00000080:00200000:2.0:1788201707.031660:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.031662:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.031670:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.031682:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.031684:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.031686:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.031695:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.031697:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.031699:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.031703:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031710:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.031717:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.031720:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.031724:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.031727:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118586, 118586] 00000080:00200000:2.0:1788201707.031738:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.031741:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.031769:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.031772:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.031788:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485732352, count: 4096 00000080:00200000:2.0:1788201707.031799:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485732352, bytes: 4096 00000080:00200000:2.0:1788201707.031802:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.031805:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.031807:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485732352, 485736448) 0 0 00000080:00200000:2.0:1788201707.031809:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.031814:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118587, 118587] 00020000:00200000:2.0:1788201707.031818:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485732352 chunk: [485732352, 485736448) 485736448, 4096 00020000:00200000:2.0:1788201707.031820:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.031849:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118587, 118587] 00000008:00010000:2.0:1788201707.031854:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.031855:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.031862:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031871:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485732352 485736447) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031880:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031888:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031900:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.031908:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485732352, 485736448) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.031913:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118587 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.031918:0:17463:0:(rw26.c:898:ll_write_end()) pos 485732352, len 4096, copied 4096 00000080:00200000:2.0:1788201707.031922:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000042b0120a[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118587] 00000080:00200000:2.0:1788201707.031926:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000057f80f29 fffffc0001029 2:0 ffff93c0c13f7000 118587 lru 00000080:00200000:2.0:1788201707.031931:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cdfd87e5 118587: 1< 2 - - > 2< 485732352 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70172 - -> 00000080:00200000:2.0:1788201707.031934:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000042b0120a 00000080:00200000:2.0:1788201707.031935:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.031937:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.031945:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.031956:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.031959:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.031961:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.031970:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.031972:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.031975:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.031979:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.031986:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.031998:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.032001:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.032005:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.032021:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118587, 118587] 00000080:00200000:2.0:1788201707.032032:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.032035:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.032064:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.032068:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.032084:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485736448, count: 4096 00000080:00200000:2.0:1788201707.032095:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485736448, bytes: 4096 00000080:00200000:2.0:1788201707.032097:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.032101:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.032103:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485736448, 485740544) 0 0 00000080:00200000:2.0:1788201707.032105:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.032110:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118588, 118588] 00020000:00200000:2.0:1788201707.032114:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485736448 chunk: [485736448, 485740544) 485740544, 4096 00020000:00200000:2.0:1788201707.032116:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.032128:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118588, 118588] 00000008:00010000:2.0:1788201707.032134:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.032136:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.032144:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032153:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485736448 485740543) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032162:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032174:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032181:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.032190:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485736448, 485740544) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.032195:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118588 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.032201:0:17463:0:(rw26.c:898:ll_write_end()) pos 485736448, len 4096, copied 4096 00000080:00200000:2.0:1788201707.032205:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e2ea8a33[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118588] 00000080:00200000:2.0:1788201707.032208:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000025a14ec4 fffffc0001029 2:0 ffff93c0c13f7a80 118588 lru 00000080:00200000:2.0:1788201707.032214:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000097a32442 118588: 1< 2 - - > 2< 485736448 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70173 - -> 00000080:00200000:2.0:1788201707.032216:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e2ea8a33 00000080:00200000:2.0:1788201707.032217:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.032220:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.032228:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.032240:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.032243:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.032245:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.032254:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.032257:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.032260:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.032264:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032277:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.032286:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.032288:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.032292:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.032295:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118588, 118588] 00000080:00200000:2.0:1788201707.032305:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.032308:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.032337:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.032340:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.032356:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485740544, count: 4096 00000080:00200000:2.0:1788201707.032367:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485740544, bytes: 4096 00000080:00200000:2.0:1788201707.032370:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.032373:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.032375:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485740544, 485744640) 0 0 00000080:00200000:2.0:1788201707.032377:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.032382:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118589, 118589] 00020000:00200000:2.0:1788201707.032386:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485740544 chunk: [485740544, 485744640) 485744640, 4096 00020000:00200000:2.0:1788201707.032388:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.032401:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118589, 118589] 00000008:00010000:2.0:1788201707.032406:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.032409:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.032417:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032426:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485740544 485744639) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032441:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032449:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032456:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.032462:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485740544, 485744640) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.032466:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118589 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.032470:0:17463:0:(rw26.c:898:ll_write_end()) pos 485740544, len 4096, copied 4096 00000080:00200000:2.0:1788201707.032473:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000072eca54f[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118589] 00000080:00200000:2.0:1788201707.032475:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000087b5fcca fffffc0001029 2:0 ffff93c0c13f7380 118589 lru 00000080:00200000:2.0:1788201707.032479:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b4477257 118589: 1< 2 - - > 2< 485740544 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70174 - -> 00000080:00200000:2.0:1788201707.032480:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000072eca54f 00000080:00200000:2.0:1788201707.032481:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.032482:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.032490:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.032502:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.032505:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.032508:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.032517:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.032520:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.032523:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.032527:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032535:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.032544:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.032547:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.032552:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.032555:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118589, 118589] 00000080:00200000:2.0:1788201707.032565:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.032568:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.032594:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.032598:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.032613:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485744640, count: 4096 00000080:00200000:2.0:1788201707.032624:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485744640, bytes: 4096 00000080:00200000:2.0:1788201707.032627:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.032631:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.032632:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485744640, 485748736) 0 0 00000080:00200000:2.0:1788201707.032635:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.032640:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118590, 118590] 00020000:00200000:2.0:1788201707.032643:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485744640 chunk: [485744640, 485748736) 485748736, 4096 00020000:00200000:2.0:1788201707.032646:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.032658:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118590, 118590] 00000008:00010000:2.0:1788201707.032663:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.032665:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.032673:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032687:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485744640 485748735) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032693:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032698:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032702:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.032706:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485744640, 485748736) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.032710:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118590 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.032714:0:17463:0:(rw26.c:898:ll_write_end()) pos 485744640, len 4096, copied 4096 00000080:00200000:2.0:1788201707.032716:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000039fa9c08[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118590] 00000080:00200000:2.0:1788201707.032718:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d54bdc1d fffffc0001029 2:0 ffff93c0c13f78c0 118590 lru 00000080:00200000:2.0:1788201707.032722:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000056673e09 118590: 1< 2 - - > 2< 485744640 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70175 - -> 00000080:00200000:2.0:1788201707.032723:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000039fa9c08 00000080:00200000:2.0:1788201707.032724:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.032725:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.032732:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.032744:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.032747:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.032750:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.032760:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.032763:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.032765:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.032769:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032778:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.032786:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.032789:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.032794:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.032797:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118590, 118590] 00000080:00200000:2.0:1788201707.032807:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.032810:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.032865:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.032869:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.032884:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485748736, count: 4096 00000080:00200000:2.0:1788201707.032894:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485748736, bytes: 4096 00000080:00200000:2.0:1788201707.032898:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.032902:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.032904:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485748736, 485752832) 0 0 00000080:00200000:2.0:1788201707.032907:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.032912:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118591, 118591] 00020000:00200000:2.0:1788201707.032915:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485748736 chunk: [485748736, 485752832) 485752832, 4096 00020000:00200000:2.0:1788201707.032917:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.032929:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118591, 118591] 00000008:00010000:2.0:1788201707.032935:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.032938:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.032946:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032955:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485748736 485752831) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032964:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032972:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.032979:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.032988:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485748736, 485752832) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.032993:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118591 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.032999:0:17463:0:(rw26.c:898:ll_write_end()) pos 485748736, len 4096, copied 4096 00000080:00200000:2.0:1788201707.033003:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a671ec6a[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118591] 00000080:00200000:2.0:1788201707.033006:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007218ff9a fffffc0001029 2:0 ffff93c0c13f8b60 118591 lru 00000080:00200000:2.0:1788201707.033026:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000795b647a 118591: 1< 2 - - > 2< 485748736 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70176 - -> 00000080:00200000:2.0:1788201707.033028:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a671ec6a 00000080:00200000:2.0:1788201707.033030:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.033032:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.033041:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.033053:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.033056:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.033058:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.033068:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.033070:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.033073:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.033077:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033085:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.033094:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.033096:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.033101:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.033104:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118591, 118591] 00000080:00200000:2.0:1788201707.033115:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.033118:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.033146:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.033150:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.033166:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485752832, count: 4096 00000080:00200000:2.0:1788201707.033176:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485752832, bytes: 4096 00000080:00200000:2.0:1788201707.033179:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.033182:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.033184:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485752832, 485756928) 0 0 00000080:00200000:2.0:1788201707.033186:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.033192:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118592, 118592] 00020000:00200000:2.0:1788201707.033195:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485752832 chunk: [485752832, 485756928) 485756928, 4096 00020000:00200000:2.0:1788201707.033197:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.033210:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118592, 118592] 00000008:00010000:2.0:1788201707.033216:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.033218:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.033226:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033235:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485752832 485756927) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033243:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033251:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033258:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.033266:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485752832, 485756928) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.033271:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118592 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.033277:0:17463:0:(rw26.c:898:ll_write_end()) pos 485752832, len 4096, copied 4096 00000080:00200000:2.0:1788201707.033281:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000093b0dc05[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118592] 00000080:00200000:2.0:1788201707.033284:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002f7cebc7 fffffc0001029 2:0 ffff93c0c13f8c40 118592 lru 00000080:00200000:2.0:1788201707.033289:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000365fffed 118592: 1< 2 - - > 2< 485752832 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70177 - -> 00000080:00200000:2.0:1788201707.033292:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000093b0dc05 00000080:00200000:2.0:1788201707.033293:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.033295:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.033303:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.033320:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.033323:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.033325:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.033334:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.033337:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.033340:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.033344:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033352:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.033360:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.033363:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.033366:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.033369:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118592, 118592] 00000080:00200000:2.0:1788201707.033379:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.033381:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.033402:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.033404:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.033414:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485756928, count: 4096 00000080:00200000:2.0:1788201707.033420:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485756928, bytes: 4096 00000080:00200000:2.0:1788201707.033422:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.033426:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.033428:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485756928, 485761024) 0 0 00000080:00200000:2.0:1788201707.033430:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.033435:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118593, 118593] 00020000:00200000:2.0:1788201707.033438:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485756928 chunk: [485756928, 485761024) 485761024, 4096 00020000:00200000:2.0:1788201707.033440:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.033451:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118593, 118593] 00000008:00010000:2.0:1788201707.033456:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.033458:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.033466:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033475:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485756928 485761023) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033482:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033487:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033491:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.033495:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485756928, 485761024) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.033499:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118593 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.033503:0:17463:0:(rw26.c:898:ll_write_end()) pos 485756928, len 4096, copied 4096 00000080:00200000:2.0:1788201707.033505:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000041442d60[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118593] 00000080:00200000:2.0:1788201707.033507:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cc1f62cc fffffc0001029 2:0 ffff93c0c13f8540 118593 lru 00000080:00200000:2.0:1788201707.033511:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009f25266f 118593: 1< 2 - - > 2< 485756928 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70178 - -> 00000080:00200000:2.0:1788201707.033512:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000041442d60 00000080:00200000:2.0:1788201707.033512:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.033514:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.033520:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.033528:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.033530:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.033531:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.033537:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.033539:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.033541:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.033543:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033548:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.033552:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.033554:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.033556:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.033558:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118593, 118593] 00000080:00200000:2.0:1788201707.033565:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.033567:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.033585:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.033587:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.033597:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485761024, count: 4096 00000080:00200000:2.0:1788201707.033603:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485761024, bytes: 4096 00000080:00200000:2.0:1788201707.033605:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.033607:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.033608:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485761024, 485765120) 0 0 00000080:00200000:2.0:1788201707.033610:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.033613:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118594, 118594] 00020000:00200000:2.0:1788201707.033615:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485761024 chunk: [485761024, 485765120) 485765120, 4096 00020000:00200000:2.0:1788201707.033617:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.033623:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118594, 118594] 00000008:00010000:2.0:1788201707.033627:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.033628:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.033633:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033638:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485761024 485765119) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033644:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033648:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033652:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.033657:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485761024, 485765120) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.033660:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118594 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.033664:0:17463:0:(rw26.c:898:ll_write_end()) pos 485761024, len 4096, copied 4096 00000080:00200000:2.0:1788201707.033666:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000eb9a576[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118594] 00000080:00200000:2.0:1788201707.033668:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000620c90ef fffffc0001029 2:0 ffff93c0c13f8460 118594 lru 00000080:00200000:2.0:1788201707.033672:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008273973b 118594: 1< 2 - - > 2< 485761024 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70179 - -> 00000080:00200000:2.0:1788201707.033674:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000eb9a576 00000080:00200000:2.0:1788201707.033675:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.033676:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.033681:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.033689:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.033691:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.033692:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.033698:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.033699:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.033701:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.033703:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033708:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.033713:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.033714:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.033717:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.033719:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118594, 118594] 00000080:00200000:2.0:1788201707.033725:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.033727:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.033743:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.033746:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.033755:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485765120, count: 4096 00000080:00200000:2.0:1788201707.033761:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485765120, bytes: 4096 00000080:00200000:2.0:1788201707.033763:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.033765:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.033766:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485765120, 485769216) 0 0 00000080:00200000:2.0:1788201707.033768:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.033771:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118595, 118595] 00020000:00200000:2.0:1788201707.033773:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485765120 chunk: [485765120, 485769216) 485769216, 4096 00020000:00200000:2.0:1788201707.033774:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.033780:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118595, 118595] 00000008:00010000:2.0:1788201707.033784:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.033785:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.033790:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033796:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485765120 485769215) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033801:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033805:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033809:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.033813:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485765120, 485769216) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.033816:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118595 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.033819:0:17463:0:(rw26.c:898:ll_write_end()) pos 485765120, len 4096, copied 4096 00000080:00200000:2.0:1788201707.033822:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d990a59c[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118595] 00000080:00200000:2.0:1788201707.033824:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000778d02e8 fffffc0001029 2:0 ffff93c0c13f80e0 118595 lru 00000080:00200000:2.0:1788201707.033827:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006658d2fc 118595: 1< 2 - - > 2< 485765120 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70180 - -> 00000080:00200000:2.0:1788201707.033842:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d990a59c 00000080:00200000:2.0:1788201707.033843:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.033844:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.033850:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.033858:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.033860:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.033861:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.033867:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.033869:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.033870:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.033873:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033877:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.033882:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.033884:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.033886:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.033888:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118595, 118595] 00000080:00200000:2.0:1788201707.033894:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.033896:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.033913:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.033915:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.033925:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485769216, count: 4096 00000080:00200000:2.0:1788201707.033931:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485769216, bytes: 4096 00000080:00200000:2.0:1788201707.033933:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.033935:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.033936:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485769216, 485773312) 0 0 00000080:00200000:2.0:1788201707.033938:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.033941:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118596, 118596] 00020000:00200000:2.0:1788201707.033943:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485769216 chunk: [485769216, 485773312) 485773312, 4096 00020000:00200000:2.0:1788201707.033944:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.033950:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118596, 118596] 00000008:00010000:2.0:1788201707.033954:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.033955:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.033960:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033966:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485769216 485773311) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033971:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033975:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.033979:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.033983:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485769216, 485773312) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.033986:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118596 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.033990:0:17463:0:(rw26.c:898:ll_write_end()) pos 485769216, len 4096, copied 4096 00000080:00200000:2.0:1788201707.033992:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fc817b75[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118596] 00000080:00200000:2.0:1788201707.033994:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007119b7f2 fffffc0001029 2:0 ffff93c0c13f8700 118596 lru 00000080:00200000:2.0:1788201707.033998:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000acd9524f 118596: 1< 2 - - > 2< 485769216 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70181 - -> 00000080:00200000:2.0:1788201707.033999:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fc817b75 00000080:00200000:2.0:1788201707.034000:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.034001:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.034006:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.034027:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.034029:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.034030:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.034037:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.034038:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.034040:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.034043:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034048:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.034052:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.034054:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.034056:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.034058:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118596, 118596] 00000080:00200000:2.0:1788201707.034065:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.034067:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.034084:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.034087:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.034097:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485773312, count: 4096 00000080:00200000:2.0:1788201707.034102:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485773312, bytes: 4096 00000080:00200000:2.0:1788201707.034104:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.034106:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.034107:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485773312, 485777408) 0 0 00000080:00200000:2.0:1788201707.034109:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.034112:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118597, 118597] 00020000:00200000:2.0:1788201707.034114:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485773312 chunk: [485773312, 485777408) 485777408, 4096 00020000:00200000:2.0:1788201707.034116:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.034122:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118597, 118597] 00000008:00010000:2.0:1788201707.034125:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.034126:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.034131:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034137:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485773312 485777407) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034142:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034147:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034151:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.034155:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485773312, 485777408) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.034158:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118597 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.034162:0:17463:0:(rw26.c:898:ll_write_end()) pos 485773312, len 4096, copied 4096 00000080:00200000:2.0:1788201707.034164:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f8a7a6c8[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118597] 00000080:00200000:2.0:1788201707.034166:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000000e49b14 fffffc0001029 2:0 ffff93c0c13f8000 118597 lru 00000080:00200000:2.0:1788201707.034170:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000aceacc6e 118597: 1< 2 - - > 2< 485773312 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70182 - -> 00000080:00200000:2.0:1788201707.034171:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f8a7a6c8 00000080:00200000:2.0:1788201707.034172:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.034173:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.034178:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.034186:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.034187:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.034189:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.034195:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.034196:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.034198:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.034200:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034205:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.034210:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.034211:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.034214:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.034216:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118597, 118597] 00000080:00200000:2.0:1788201707.034222:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.034224:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.034241:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.034244:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.034253:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485777408, count: 4096 00000080:00200000:2.0:1788201707.034259:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485777408, bytes: 4096 00000080:00200000:2.0:1788201707.034261:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.034263:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.034264:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485777408, 485781504) 0 0 00000080:00200000:2.0:1788201707.034266:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.034269:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118598, 118598] 00020000:00200000:2.0:1788201707.034271:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485777408 chunk: [485777408, 485781504) 485781504, 4096 00020000:00200000:2.0:1788201707.034272:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.034283:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118598, 118598] 00000008:00010000:2.0:1788201707.034288:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.034290:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.034297:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034307:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485777408 485781503) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034316:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034323:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034330:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.034344:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485777408, 485781504) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.034349:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118598 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.034353:0:17463:0:(rw26.c:898:ll_write_end()) pos 485777408, len 4096, copied 4096 00000080:00200000:2.0:1788201707.034357:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001e5de0f7[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118598] 00000080:00200000:2.0:1788201707.034360:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ff42b612 fffffc0001029 2:0 ffff93c0c13f8a80 118598 lru 00000080:00200000:2.0:1788201707.034366:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cbd2d964 118598: 1< 2 - - > 2< 485777408 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70183 - -> 00000080:00200000:2.0:1788201707.034368:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001e5de0f7 00000080:00200000:2.0:1788201707.034370:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.034371:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.034379:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.034389:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.034391:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.034392:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.034399:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.034401:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.034402:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.034405:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034410:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.034415:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.034416:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.034420:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.034422:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118598, 118598] 00000080:00200000:2.0:1788201707.034432:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.034435:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.034463:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.034466:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.034480:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485781504, count: 4096 00000080:00200000:2.0:1788201707.034487:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485781504, bytes: 4096 00000080:00200000:2.0:1788201707.034489:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.034491:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.034492:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485781504, 485785600) 0 0 00000080:00200000:2.0:1788201707.034494:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.034497:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118599, 118599] 00020000:00200000:2.0:1788201707.034499:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485781504 chunk: [485781504, 485785600) 485785600, 4096 00020000:00200000:2.0:1788201707.034501:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.034508:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118599, 118599] 00000008:00010000:2.0:1788201707.034511:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.034512:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.034518:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034523:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485781504 485785599) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034529:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034533:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034541:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.034545:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485781504, 485785600) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.034548:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118599 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.034551:0:17463:0:(rw26.c:898:ll_write_end()) pos 485781504, len 4096, copied 4096 00000080:00200000:2.0:1788201707.034554:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000aab590d0[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118599] 00000080:00200000:2.0:1788201707.034555:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c04ad236 fffffc0001029 2:0 ffff93c0c13f8380 118599 lru 00000080:00200000:2.0:1788201707.034559:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008420b173 118599: 1< 2 - - > 2< 485781504 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70184 - -> 00000080:00200000:2.0:1788201707.034560:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000aab590d0 00000080:00200000:2.0:1788201707.034561:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.034562:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.034568:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.034575:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.034577:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.034578:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.034584:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.034586:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.034587:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.034590:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034595:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.034604:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.034605:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.034608:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.034610:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118599, 118599] 00000080:00200000:2.0:1788201707.034616:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.034618:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.034635:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.034637:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.034647:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485785600, count: 4096 00000080:00200000:2.0:1788201707.034653:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485785600, bytes: 4096 00000080:00200000:2.0:1788201707.034654:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.034657:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.034658:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485785600, 485789696) 0 0 00000080:00200000:2.0:1788201707.034659:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.034662:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118600, 118600] 00020000:00200000:2.0:1788201707.034664:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485785600 chunk: [485785600, 485789696) 485789696, 4096 00020000:00200000:2.0:1788201707.034666:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.034672:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118600, 118600] 00000008:00010000:2.0:1788201707.034675:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.034676:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.034681:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034687:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485785600 485789695) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034692:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034700:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034704:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.034708:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485785600, 485789696) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.034711:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118600 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.034714:0:17463:0:(rw26.c:898:ll_write_end()) pos 485785600, len 4096, copied 4096 00000080:00200000:2.0:1788201707.034717:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b85f73a5[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118600] 00000080:00200000:2.0:1788201707.034718:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000bcdc51d fffffc0001029 2:0 ffff93c0c13f88c0 118600 lru 00000080:00200000:2.0:1788201707.034722:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000a296a77 118600: 1< 2 - - > 2< 485785600 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70185 - -> 00000080:00200000:2.0:1788201707.034723:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b85f73a5 00000080:00200000:2.0:1788201707.034723:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.034725:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.034730:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.034737:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.034739:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.034740:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.034746:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.034747:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.034749:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.034751:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034759:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.034764:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.034765:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.034768:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.034770:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118600, 118600] 00000080:00200000:2.0:1788201707.034776:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.034778:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.034795:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.034797:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.034806:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485789696, count: 4096 00000080:00200000:2.0:1788201707.034812:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485789696, bytes: 4096 00000080:00200000:2.0:1788201707.034814:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.034816:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.034817:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485789696, 485793792) 0 0 00000080:00200000:2.0:1788201707.034819:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.034821:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118601, 118601] 00020000:00200000:2.0:1788201707.034823:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485789696 chunk: [485789696, 485793792) 485793792, 4096 00020000:00200000:2.0:1788201707.034825:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.034845:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118601, 118601] 00000008:00010000:2.0:1788201707.034848:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.034850:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.034855:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034860:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485789696 485793791) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034869:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034874:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034878:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.034882:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485789696, 485793792) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.034885:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118601 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.034889:0:17463:0:(rw26.c:898:ll_write_end()) pos 485789696, len 4096, copied 4096 00000080:00200000:2.0:1788201707.034891:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005d200fc6[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118601] 00000080:00200000:2.0:1788201707.034892:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000df07b760 fffffc0001029 2:0 ffff93c0c13f8ee0 118601 lru 00000080:00200000:2.0:1788201707.034896:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000014436311 118601: 1< 2 - - > 2< 485789696 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70186 - -> 00000080:00200000:2.0:1788201707.034897:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005d200fc6 00000080:00200000:2.0:1788201707.034898:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.034899:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.034904:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.034912:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.034913:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.034915:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.034920:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.034922:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.034924:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.034926:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.034931:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.034936:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.034937:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.034940:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.034942:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118601, 118601] 00000080:00200000:2.0:1788201707.034948:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.034950:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.034967:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.034969:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.034978:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485793792, count: 4096 00000080:00200000:2.0:1788201707.034984:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485793792, bytes: 4096 00000080:00200000:2.0:1788201707.034986:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.034988:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.034989:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485793792, 485797888) 0 0 00000080:00200000:2.0:1788201707.034991:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.034994:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118602, 118602] 00020000:00200000:2.0:1788201707.034996:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485793792 chunk: [485793792, 485797888) 485797888, 4096 00020000:00200000:2.0:1788201707.034997:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.035004:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118602, 118602] 00000008:00010000:2.0:1788201707.035007:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.035021:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.035028:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035038:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485793792 485797887) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035043:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035048:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035052:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.035056:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485793792, 485797888) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.035060:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118602 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.035063:0:17463:0:(rw26.c:898:ll_write_end()) pos 485793792, len 4096, copied 4096 00000080:00200000:2.0:1788201707.035066:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000cdb016cc[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118602] 00000080:00200000:2.0:1788201707.035067:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000001cecc1a fffffc0001029 2:0 ffff93c0c13f87e0 118602 lru 00000080:00200000:2.0:1788201707.035070:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000013a9f9af 118602: 1< 2 - - > 2< 485793792 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70187 - -> 00000080:00200000:2.0:1788201707.035072:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000cdb016cc 00000080:00200000:2.0:1788201707.035072:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.035074:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.035079:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.035087:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.035089:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.035090:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.035096:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.035098:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.035100:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.035102:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035107:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.035112:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.035113:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.035116:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.035117:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118602, 118602] 00000080:00200000:2.0:1788201707.035124:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.035126:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.035143:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.035145:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.035155:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485797888, count: 4096 00000080:00200000:2.0:1788201707.035161:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485797888, bytes: 4096 00000080:00200000:2.0:1788201707.035163:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.035165:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.035166:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485797888, 485801984) 0 0 00000080:00200000:2.0:1788201707.035167:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.035171:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118603, 118603] 00020000:00200000:2.0:1788201707.035173:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485797888 chunk: [485797888, 485801984) 485801984, 4096 00020000:00200000:2.0:1788201707.035174:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.035181:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118603, 118603] 00000008:00010000:2.0:1788201707.035184:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.035185:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.035190:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035196:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485797888 485801983) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035201:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035205:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035209:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.035213:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485797888, 485801984) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.035216:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118603 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.035219:0:17463:0:(rw26.c:898:ll_write_end()) pos 485797888, len 4096, copied 4096 00000080:00200000:2.0:1788201707.035221:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001be24a3f[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118603] 00000080:00200000:2.0:1788201707.035223:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000514bd9e9 fffffc0001029 2:0 ffff93c0c13f82a0 118603 lru 00000080:00200000:2.0:1788201707.035227:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000da4327f3 118603: 1< 2 - - > 2< 485797888 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70188 - -> 00000080:00200000:2.0:1788201707.035228:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001be24a3f 00000080:00200000:2.0:1788201707.035229:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.035230:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.035235:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.035243:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.035245:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.035246:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.035252:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.035253:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.035255:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.035257:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035262:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.035266:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.035268:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.035271:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.035273:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118603, 118603] 00000080:00200000:2.0:1788201707.035283:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.035286:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.035314:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.035317:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.035332:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485801984, count: 4096 00000080:00200000:2.0:1788201707.035342:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485801984, bytes: 4096 00000080:00200000:2.0:1788201707.035345:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.035348:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.035350:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485801984, 485806080) 0 0 00000080:00200000:2.0:1788201707.035352:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.035357:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118604, 118604] 00020000:00200000:2.0:1788201707.035360:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485801984 chunk: [485801984, 485806080) 485806080, 4096 00020000:00200000:2.0:1788201707.035362:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.035370:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118604, 118604] 00000008:00010000:2.0:1788201707.035373:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.035375:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.035380:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035386:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485801984 485806079) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035391:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035396:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035400:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.035404:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485801984, 485806080) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.035408:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118604 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.035411:0:17463:0:(rw26.c:898:ll_write_end()) pos 485801984, len 4096, copied 4096 00000080:00200000:2.0:1788201707.035414:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008adf9b85[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118604] 00000080:00200000:2.0:1788201707.035415:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000f9ac3d6 fffffc0001029 2:0 ffff93c0c13f8e00 118604 lru 00000080:00200000:2.0:1788201707.035419:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003ea405a2 118604: 1< 2 - - > 2< 485801984 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70189 - -> 00000080:00200000:2.0:1788201707.035421:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008adf9b85 00000080:00200000:2.0:1788201707.035422:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.035424:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.035432:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.035448:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.035451:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.035453:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.035463:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.035465:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.035468:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.035472:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035479:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.035484:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.035486:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.035489:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.035491:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118604, 118604] 00000080:00200000:2.0:1788201707.035498:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.035500:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.035519:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.035521:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.035531:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485806080, count: 4096 00000080:00200000:2.0:1788201707.035537:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485806080, bytes: 4096 00000080:00200000:2.0:1788201707.035539:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.035541:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.035542:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485806080, 485810176) 0 0 00000080:00200000:2.0:1788201707.035543:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.035547:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118605, 118605] 00020000:00200000:2.0:1788201707.035549:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485806080 chunk: [485806080, 485810176) 485810176, 4096 00020000:00200000:2.0:1788201707.035550:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.035557:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118605, 118605] 00000008:00010000:2.0:1788201707.035560:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.035561:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.035566:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035572:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485806080 485810175) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035577:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035582:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035586:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.035590:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485806080, 485810176) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.035593:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118605 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.035597:0:17463:0:(rw26.c:898:ll_write_end()) pos 485806080, len 4096, copied 4096 00000080:00200000:2.0:1788201707.035600:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001285ae80[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118605] 00000080:00200000:2.0:1788201707.035601:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000057d22da3 fffffc0001029 2:0 ffff93c0c13f81c0 118605 lru 00000080:00200000:2.0:1788201707.035605:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008b1a8354 118605: 1< 2 - - > 2< 485806080 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70190 - -> 00000080:00200000:2.0:1788201707.035606:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001285ae80 00000080:00200000:2.0:1788201707.035607:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.035609:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.035614:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.035622:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.035623:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.035625:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.035630:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.035632:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.035634:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.035636:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035640:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.035645:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.035646:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.035649:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.035651:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118605, 118605] 00000080:00200000:2.0:1788201707.035657:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.035659:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.035676:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.035679:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.035688:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485810176, count: 4096 00000080:00200000:2.0:1788201707.035694:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485810176, bytes: 4096 00000080:00200000:2.0:1788201707.035696:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.035698:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.035699:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485810176, 485814272) 0 0 00000080:00200000:2.0:1788201707.035701:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.035704:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118606, 118606] 00020000:00200000:2.0:1788201707.035706:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485810176 chunk: [485810176, 485814272) 485814272, 4096 00020000:00200000:2.0:1788201707.035707:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.035713:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118606, 118606] 00000008:00010000:2.0:1788201707.035717:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.035718:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.035723:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035729:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485810176 485814271) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035734:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035739:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035743:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.035747:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485810176, 485814272) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.035750:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118606 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.035753:0:17463:0:(rw26.c:898:ll_write_end()) pos 485810176, len 4096, copied 4096 00000080:00200000:2.0:1788201707.035756:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008c5d32f7[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118606] 00000080:00200000:2.0:1788201707.035758:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d6ea2618 fffffc0001029 2:0 ffff93c0c13f8620 118606 lru 00000080:00200000:2.0:1788201707.035761:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000030f6aa68 118606: 1< 2 - - > 2< 485810176 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70191 - -> 00000080:00200000:2.0:1788201707.035764:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008c5d32f7 00000080:00200000:2.0:1788201707.035765:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.035766:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.035771:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.035778:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.035780:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.035781:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.035787:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.035788:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.035790:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.035792:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035796:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.035801:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.035802:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.035805:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.035807:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118606, 118606] 00000080:00200000:2.0:1788201707.035813:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.035815:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.035849:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.035851:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.035862:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485814272, count: 4096 00000080:00200000:2.0:1788201707.035869:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485814272, bytes: 4096 00000080:00200000:2.0:1788201707.035872:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.035874:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.035875:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485814272, 485818368) 0 0 00000080:00200000:2.0:1788201707.035876:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.035879:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118607, 118607] 00020000:00200000:2.0:1788201707.035881:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485814272 chunk: [485814272, 485818368) 485818368, 4096 00020000:00200000:2.0:1788201707.035883:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.035889:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118607, 118607] 00000008:00010000:2.0:1788201707.035892:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.035893:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.035898:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035904:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485814272 485818367) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035910:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035914:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035918:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.035923:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485814272, 485818368) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.035925:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118607 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.035929:0:17463:0:(rw26.c:898:ll_write_end()) pos 485814272, len 4096, copied 4096 00000080:00200000:2.0:1788201707.035931:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000026b2f317[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118607] 00000080:00200000:2.0:1788201707.035934:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000466465bd fffffc0001029 2:0 ffff93c0c13f8d20 118607 lru 00000080:00200000:2.0:1788201707.035937:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b7dea7a6 118607: 1< 2 - - > 2< 485814272 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70192 - -> 00000080:00200000:2.0:1788201707.035939:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000026b2f317 00000080:00200000:2.0:1788201707.035939:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.035940:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.035946:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.035953:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.035955:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.035956:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.035962:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.035963:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.035965:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.035967:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.035972:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.035977:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.035978:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.035981:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.035983:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118607, 118607] 00000080:00200000:2.0:1788201707.035989:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.035991:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.036020:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.036023:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.036034:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485818368, count: 4096 00000080:00200000:2.0:1788201707.036041:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485818368, bytes: 4096 00000080:00200000:2.0:1788201707.036042:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.036044:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.036045:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485818368, 485822464) 0 0 00000080:00200000:2.0:1788201707.036047:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.036050:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118608, 118608] 00020000:00200000:2.0:1788201707.036052:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485818368 chunk: [485818368, 485822464) 485822464, 4096 00020000:00200000:2.0:1788201707.036054:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.036061:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118608, 118608] 00000008:00010000:2.0:1788201707.036064:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.036065:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.036070:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036076:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485818368 485822463) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036082:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036086:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036090:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.036095:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485818368, 485822464) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.036098:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118608 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.036102:0:17463:0:(rw26.c:898:ll_write_end()) pos 485818368, len 4096, copied 4096 00000080:00200000:2.0:1788201707.036104:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008818db1d[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118608] 00000080:00200000:2.0:1788201707.036106:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008058487d fffffc0001029 2:0 ffff93c0c13f89a0 118608 lru 00000080:00200000:2.0:1788201707.036110:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000053031a96 118608: 1< 2 - - > 2< 485818368 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70193 - -> 00000080:00200000:2.0:1788201707.036111:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008818db1d 00000080:00200000:2.0:1788201707.036112:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.036113:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.036118:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.036127:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.036128:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.036130:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.036136:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.036137:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.036139:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.036141:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036146:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.036151:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.036152:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.036155:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.036157:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118608, 118608] 00000080:00200000:2.0:1788201707.036163:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.036165:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.036182:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.036184:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.036194:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485822464, count: 4096 00000080:00200000:2.0:1788201707.036200:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485822464, bytes: 4096 00000080:00200000:2.0:1788201707.036202:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.036204:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.036205:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485822464, 485826560) 0 0 00000080:00200000:2.0:1788201707.036206:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.036209:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118609, 118609] 00020000:00200000:2.0:1788201707.036211:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485822464 chunk: [485822464, 485826560) 485826560, 4096 00020000:00200000:2.0:1788201707.036213:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.036219:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118609, 118609] 00000008:00010000:2.0:1788201707.036222:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.036223:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.036228:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036234:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485822464 485826559) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036240:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036244:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036248:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.036253:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485822464, 485826560) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.036256:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118609 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.036260:0:17463:0:(rw26.c:898:ll_write_end()) pos 485822464, len 4096, copied 4096 00000080:00200000:2.0:1788201707.036262:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000605c2f88[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118609] 00000080:00200000:2.0:1788201707.036264:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000dc8744f fffffc0001029 2:0 ffff93c0c13f9700 118609 lru 00000080:00200000:2.0:1788201707.036267:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002f58e4d2 118609: 1< 2 - - > 2< 485822464 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70194 - -> 00000080:00200000:2.0:1788201707.036269:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000605c2f88 00000080:00200000:2.0:1788201707.036270:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.036271:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.036276:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.036284:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.036285:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.036287:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.036292:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.036294:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.036296:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.036298:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036302:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.036307:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.036309:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.036311:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.036313:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118609, 118609] 00000080:00200000:2.0:1788201707.036319:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.036321:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.036339:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.036341:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.036351:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485826560, count: 4096 00000080:00200000:2.0:1788201707.036356:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485826560, bytes: 4096 00000080:00200000:2.0:1788201707.036358:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.036361:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.036362:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485826560, 485830656) 0 0 00000080:00200000:2.0:1788201707.036363:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.036366:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118610, 118610] 00020000:00200000:2.0:1788201707.036368:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485826560 chunk: [485826560, 485830656) 485830656, 4096 00020000:00200000:2.0:1788201707.036370:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.036376:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118610, 118610] 00000008:00010000:2.0:1788201707.036379:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.036380:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.036385:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036391:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485826560 485830655) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036396:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036400:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036404:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.036411:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485826560, 485830656) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.036414:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118610 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.036418:0:17463:0:(rw26.c:898:ll_write_end()) pos 485826560, len 4096, copied 4096 00000080:00200000:2.0:1788201707.036420:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000464dd5b7[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118610] 00000080:00200000:2.0:1788201707.036423:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000070a94abd fffffc0001029 2:0 ffff93c0c13f9000 118610 lru 00000080:00200000:2.0:1788201707.036428:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bf13bead 118610: 1< 2 - - > 2< 485826560 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70195 - -> 00000080:00200000:2.0:1788201707.036430:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000464dd5b7 00000080:00200000:2.0:1788201707.036431:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.036433:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.036441:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.036453:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.036456:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.036458:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.036467:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.036470:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.036472:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.036476:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036482:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.036487:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.036489:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.036492:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.036494:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118610, 118610] 00000080:00200000:2.0:1788201707.036501:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.036503:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.036522:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.036524:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.036534:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485830656, count: 4096 00000080:00200000:2.0:1788201707.036540:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485830656, bytes: 4096 00000080:00200000:2.0:1788201707.036542:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.036544:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.036545:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485830656, 485834752) 0 0 00000080:00200000:2.0:1788201707.036546:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.036550:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118611, 118611] 00020000:00200000:2.0:1788201707.036552:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485830656 chunk: [485830656, 485834752) 485834752, 4096 00020000:00200000:2.0:1788201707.036553:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.036560:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118611, 118611] 00000008:00010000:2.0:1788201707.036563:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.036564:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.036569:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036575:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485830656 485834751) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036580:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036585:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036592:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.036597:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485830656, 485834752) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.036600:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118611 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.036603:0:17463:0:(rw26.c:898:ll_write_end()) pos 485830656, len 4096, copied 4096 00000080:00200000:2.0:1788201707.036605:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d314c57d[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118611] 00000080:00200000:2.0:1788201707.036607:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009dd5e57d fffffc0001029 2:0 ffff93c0c13f9a80 118611 lru 00000080:00200000:2.0:1788201707.036610:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000404523fa 118611: 1< 2 - - > 2< 485830656 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70196 - -> 00000080:00200000:2.0:1788201707.036612:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d314c57d 00000080:00200000:2.0:1788201707.036612:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.036614:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.036619:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.036627:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.036628:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.036630:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.036636:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.036637:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.036639:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.036641:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036646:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.036654:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.036656:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.036658:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.036660:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118611, 118611] 00000080:00200000:2.0:1788201707.036667:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.036668:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.036686:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.036688:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.036697:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485834752, count: 4096 00000080:00200000:2.0:1788201707.036703:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485834752, bytes: 4096 00000080:00200000:2.0:1788201707.036705:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.036707:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.036708:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485834752, 485838848) 0 0 00000080:00200000:2.0:1788201707.036709:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.036713:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118612, 118612] 00020000:00200000:2.0:1788201707.036715:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485834752 chunk: [485834752, 485838848) 485838848, 4096 00020000:00200000:2.0:1788201707.036716:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.036722:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118612, 118612] 00000008:00010000:2.0:1788201707.036725:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.036727:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.036731:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036737:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485834752 485838847) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036742:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036751:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036755:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.036759:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485834752, 485838848) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.036762:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118612 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.036765:0:17463:0:(rw26.c:898:ll_write_end()) pos 485834752, len 4096, copied 4096 00000080:00200000:2.0:1788201707.036767:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000928bd96a[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118612] 00000080:00200000:2.0:1788201707.036769:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008f4510b9 fffffc0001029 2:0 ffff93c0c13f9380 118612 lru 00000080:00200000:2.0:1788201707.036772:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000008344516 118612: 1< 2 - - > 2< 485834752 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70197 - -> 00000080:00200000:2.0:1788201707.036773:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000928bd96a 00000080:00200000:2.0:1788201707.036774:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.036775:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.036780:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.036788:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.036789:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.036791:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.036796:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.036798:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.036800:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.036802:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036810:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.036815:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.036816:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.036819:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.036820:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118612, 118612] 00000080:00200000:2.0:1788201707.036827:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.036844:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.036866:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.036868:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.036878:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485838848, count: 4096 00000080:00200000:2.0:1788201707.036884:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485838848, bytes: 4096 00000080:00200000:2.0:1788201707.036886:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.036888:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.036889:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485838848, 485842944) 0 0 00000080:00200000:2.0:1788201707.036891:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.036894:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118613, 118613] 00020000:00200000:2.0:1788201707.036896:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485838848 chunk: [485838848, 485842944) 485842944, 4096 00020000:00200000:2.0:1788201707.036897:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.036904:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118613, 118613] 00000008:00010000:2.0:1788201707.036907:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.036908:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.036913:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036919:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485838848 485842943) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036927:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036932:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036936:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.036940:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485838848, 485842944) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.036943:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118613 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.036947:0:17463:0:(rw26.c:898:ll_write_end()) pos 485838848, len 4096, copied 4096 00000080:00200000:2.0:1788201707.036949:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007f47fe91[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118613] 00000080:00200000:2.0:1788201707.036950:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000796dc1ba fffffc0001029 2:0 ffff93c0c13f98c0 118613 lru 00000080:00200000:2.0:1788201707.036954:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c98aa4dd 118613: 1< 2 - - > 2< 485838848 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70198 - -> 00000080:00200000:2.0:1788201707.036955:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007f47fe91 00000080:00200000:2.0:1788201707.036956:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.036957:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.036962:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.036970:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.036972:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.036973:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.036979:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.036980:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.036983:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.036985:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.036990:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.036995:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.036996:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.036999:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.037001:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118613, 118613] 00000080:00200000:2.0:1788201707.037018:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.037021:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.037049:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.037051:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.037061:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485842944, count: 4096 00000080:00200000:2.0:1788201707.037067:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485842944, bytes: 4096 00000080:00200000:2.0:1788201707.037069:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.037071:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.037072:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485842944, 485847040) 0 0 00000080:00200000:2.0:1788201707.037074:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.037077:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118614, 118614] 00020000:00200000:2.0:1788201707.037079:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485842944 chunk: [485842944, 485847040) 485847040, 4096 00020000:00200000:2.0:1788201707.037080:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.037087:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118614, 118614] 00000008:00010000:2.0:1788201707.037090:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.037092:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.037097:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037106:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485842944 485847039) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037112:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037116:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037120:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.037125:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485842944, 485847040) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.037128:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118614 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.037132:0:17463:0:(rw26.c:898:ll_write_end()) pos 485842944, len 4096, copied 4096 00000080:00200000:2.0:1788201707.037134:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f6a9332b[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118614] 00000080:00200000:2.0:1788201707.037136:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fb746942 fffffc0001029 2:0 ffff93c0c13f9ee0 118614 lru 00000080:00200000:2.0:1788201707.037139:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004e1278a8 118614: 1< 2 - - > 2< 485842944 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70199 - -> 00000080:00200000:2.0:1788201707.037141:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f6a9332b 00000080:00200000:2.0:1788201707.037142:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.037143:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.037148:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.037156:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.037157:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.037159:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.037165:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.037167:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.037168:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.037170:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037175:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.037180:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.037182:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.037184:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.037186:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118614, 118614] 00000080:00200000:2.0:1788201707.037193:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.037195:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.037212:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.037214:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.037224:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485847040, count: 4096 00000080:00200000:2.0:1788201707.037233:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485847040, bytes: 4096 00000080:00200000:2.0:1788201707.037236:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.037240:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.037241:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485847040, 485851136) 0 0 00000080:00200000:2.0:1788201707.037244:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.037249:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118615, 118615] 00020000:00200000:2.0:1788201707.037252:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485847040 chunk: [485847040, 485851136) 485851136, 4096 00020000:00200000:2.0:1788201707.037254:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.037265:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118615, 118615] 00000008:00010000:2.0:1788201707.037270:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.037272:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.037279:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037287:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485847040 485851135) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037295:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037301:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037307:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.037313:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485847040, 485851136) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.037318:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118615 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.037323:0:17463:0:(rw26.c:898:ll_write_end()) pos 485847040, len 4096, copied 4096 00000080:00200000:2.0:1788201707.037326:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001209c4df[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118615] 00000080:00200000:2.0:1788201707.037329:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000001006095 fffffc0001029 2:0 ffff93c0c13f97e0 118615 lru 00000080:00200000:2.0:1788201707.037334:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cb8db3d7 118615: 1< 2 - - > 2< 485847040 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70200 - -> 00000080:00200000:2.0:1788201707.037336:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001209c4df 00000080:00200000:2.0:1788201707.037337:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.037339:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.037347:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.037358:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.037361:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.037363:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.037372:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.037374:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.037377:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.037381:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037389:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.037396:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.037398:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.037402:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.037405:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118615, 118615] 00000080:00200000:2.0:1788201707.037414:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.037417:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.037444:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.037448:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.037462:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485851136, count: 4096 00000080:00200000:2.0:1788201707.037471:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485851136, bytes: 4096 00000080:00200000:2.0:1788201707.037473:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.037477:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.037478:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485851136, 485855232) 0 0 00000080:00200000:2.0:1788201707.037480:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.037485:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118616, 118616] 00020000:00200000:2.0:1788201707.037488:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485851136 chunk: [485851136, 485855232) 485855232, 4096 00020000:00200000:2.0:1788201707.037490:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.037501:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118616, 118616] 00000008:00010000:2.0:1788201707.037507:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.037509:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.037516:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037524:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485851136 485855231) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037534:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037541:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037549:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.037556:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485851136, 485855232) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.037561:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118616 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.037566:0:17463:0:(rw26.c:898:ll_write_end()) pos 485851136, len 4096, copied 4096 00000080:00200000:2.0:1788201707.037570:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d0a75dc7[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118616] 00000080:00200000:2.0:1788201707.037572:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002b2fc4ce fffffc0001029 2:0 ffff93c0c13f92a0 118616 lru 00000080:00200000:2.0:1788201707.037577:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002e9999cb 118616: 1< 2 - - > 2< 485851136 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70201 - -> 00000080:00200000:2.0:1788201707.037579:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d0a75dc7 00000080:00200000:2.0:1788201707.037580:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.037582:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.037590:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.037608:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.037610:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.037613:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.037622:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.037624:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.037626:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.037630:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037637:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.037645:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.037648:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.037652:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.037655:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118616, 118616] 00000080:00200000:2.0:1788201707.037666:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.037669:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.037696:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.037700:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.037716:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485855232, count: 4096 00000080:00200000:2.0:1788201707.037727:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485855232, bytes: 4096 00000080:00200000:2.0:1788201707.037730:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.037734:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.037735:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485855232, 485859328) 0 0 00000080:00200000:2.0:1788201707.037738:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.037742:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118617, 118617] 00020000:00200000:2.0:1788201707.037745:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485855232 chunk: [485855232, 485859328) 485859328, 4096 00020000:00200000:2.0:1788201707.037748:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.037759:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118617, 118617] 00000008:00010000:2.0:1788201707.037764:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.037766:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.037774:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037782:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485855232 485859327) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037790:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037797:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037804:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.037812:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485855232, 485859328) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.037817:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118617 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.037822:0:17463:0:(rw26.c:898:ll_write_end()) pos 485855232, len 4096, copied 4096 00000080:00200000:2.0:1788201707.037826:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001916647a[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118617] 00000080:00200000:2.0:1788201707.037844:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000064c7a6d6 fffffc0001029 2:0 ffff93c0c13f9e00 118617 lru 00000080:00200000:2.0:1788201707.037851:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000042153a6d 118617: 1< 2 - - > 2< 485855232 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70202 - -> 00000080:00200000:2.0:1788201707.037853:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001916647a 00000080:00200000:2.0:1788201707.037854:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.037857:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.037866:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.037878:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.037881:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.037883:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.037892:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.037895:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.037897:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.037901:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.037909:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.037917:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.037919:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.037924:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.037927:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118617, 118617] 00000080:00200000:2.0:1788201707.037938:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.037941:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.037968:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.037972:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.037988:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485859328, count: 4096 00000080:00200000:2.0:1788201707.037999:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485859328, bytes: 4096 00000080:00200000:2.0:1788201707.038002:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.038005:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.038007:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485859328, 485863424) 0 0 00000080:00200000:2.0:1788201707.038022:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.038027:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118618, 118618] 00020000:00200000:2.0:1788201707.038031:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485859328 chunk: [485859328, 485863424) 485863424, 4096 00020000:00200000:2.0:1788201707.038033:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.038047:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118618, 118618] 00000008:00010000:2.0:1788201707.038052:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.038054:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.038062:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038071:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485859328 485863423) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038079:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038087:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038094:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.038102:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485859328, 485863424) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.038107:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118618 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.038113:0:17463:0:(rw26.c:898:ll_write_end()) pos 485859328, len 4096, copied 4096 00000080:00200000:2.0:1788201707.038117:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f0e0d672[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118618] 00000080:00200000:2.0:1788201707.038120:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000dab3d8b7 fffffc0001029 2:0 ffff93c0c13f91c0 118618 lru 00000080:00200000:2.0:1788201707.038126:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000cca86db 118618: 1< 2 - - > 2< 485859328 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70203 - -> 00000080:00200000:2.0:1788201707.038129:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f0e0d672 00000080:00200000:2.0:1788201707.038130:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.038132:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.038141:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.038153:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.038156:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.038158:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.038167:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.038169:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.038172:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.038177:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038185:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.038192:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.038194:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.038199:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.038202:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118618, 118618] 00000080:00200000:2.0:1788201707.038212:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.038215:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.038242:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.038245:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.038261:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485863424, count: 4096 00000080:00200000:2.0:1788201707.038271:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485863424, bytes: 4096 00000080:00200000:2.0:1788201707.038275:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.038278:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.038280:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485863424, 485867520) 0 0 00000080:00200000:2.0:1788201707.038282:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.038287:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118619, 118619] 00020000:00200000:2.0:1788201707.038290:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485863424 chunk: [485863424, 485867520) 485867520, 4096 00020000:00200000:2.0:1788201707.038292:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.038305:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118619, 118619] 00000008:00010000:2.0:1788201707.038310:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.038313:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.038320:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038329:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485863424 485867519) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038339:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038348:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038354:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.038362:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485863424, 485867520) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.038367:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118619 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.038372:0:17463:0:(rw26.c:898:ll_write_end()) pos 485863424, len 4096, copied 4096 00000080:00200000:2.0:1788201707.038375:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a7f8efdd[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118619] 00000080:00200000:2.0:1788201707.038379:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f31f92d6 fffffc0001029 2:0 ffff93c0c13f9620 118619 lru 00000080:00200000:2.0:1788201707.038384:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f95af045 118619: 1< 2 - - > 2< 485863424 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70204 - -> 00000080:00200000:2.0:1788201707.038387:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a7f8efdd 00000080:00200000:2.0:1788201707.038388:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.038390:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.038398:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.038410:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.038413:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.038415:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.038423:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.038426:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.038429:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.038433:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038441:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.038450:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.038453:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.038457:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.038460:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118619, 118619] 00000080:00200000:2.0:1788201707.038471:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.038474:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.038504:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.038507:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.038524:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485867520, count: 4096 00000080:00200000:2.0:1788201707.038534:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485867520, bytes: 4096 00000080:00200000:2.0:1788201707.038537:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.038541:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.038542:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485867520, 485871616) 0 0 00000080:00200000:2.0:1788201707.038545:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.038550:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118620, 118620] 00020000:00200000:2.0:1788201707.038553:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485867520 chunk: [485867520, 485871616) 485871616, 4096 00020000:00200000:2.0:1788201707.038556:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.038568:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118620, 118620] 00000008:00010000:2.0:1788201707.038573:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.038575:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.038583:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038593:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485867520 485871615) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038602:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038608:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038616:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.038623:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485867520, 485871616) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.038628:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118620 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.038634:0:17463:0:(rw26.c:898:ll_write_end()) pos 485867520, len 4096, copied 4096 00000080:00200000:2.0:1788201707.038638:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001b04fb75[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118620] 00000080:00200000:2.0:1788201707.038641:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000023d95d89 fffffc0001029 2:0 ffff93c0c13f9d20 118620 lru 00000080:00200000:2.0:1788201707.038646:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009d7054c3 118620: 1< 2 - - > 2< 485867520 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70205 - -> 00000080:00200000:2.0:1788201707.038648:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001b04fb75 00000080:00200000:2.0:1788201707.038650:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.038651:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.038660:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.038673:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.038676:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.038678:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.038687:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.038690:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.038693:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.038697:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038705:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.038712:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.038714:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.038718:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.038721:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118620, 118620] 00000080:00200000:2.0:1788201707.038731:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.038734:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.038761:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.038764:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.038779:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485871616, count: 4096 00000080:00200000:2.0:1788201707.038790:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485871616, bytes: 4096 00000080:00200000:2.0:1788201707.038793:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.038797:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.038798:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485871616, 485875712) 0 0 00000080:00200000:2.0:1788201707.038801:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.038806:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118621, 118621] 00020000:00200000:2.0:1788201707.038808:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485871616 chunk: [485871616, 485875712) 485875712, 4096 00020000:00200000:2.0:1788201707.038811:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.038822:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118621, 118621] 00000008:00010000:2.0:1788201707.038827:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.038844:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.038852:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038861:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485871616 485875711) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038869:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038876:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038883:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.038890:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485871616, 485875712) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.038896:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118621 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.038902:0:17463:0:(rw26.c:898:ll_write_end()) pos 485871616, len 4096, copied 4096 00000080:00200000:2.0:1788201707.038906:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009c571239[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118621] 00000080:00200000:2.0:1788201707.038909:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000125654f9 fffffc0001029 2:0 ffff93c0c13f99a0 118621 lru 00000080:00200000:2.0:1788201707.038915:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000608f9712 118621: 1< 2 - - > 2< 485871616 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70206 - -> 00000080:00200000:2.0:1788201707.038917:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009c571239 00000080:00200000:2.0:1788201707.038919:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.038921:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.038929:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.038941:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.038944:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.038947:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.038955:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.038958:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.038961:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.038964:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.038972:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.038980:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.038983:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.038987:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.038990:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118621, 118621] 00000080:00200000:2.0:1788201707.039001:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.039004:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.039042:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.039045:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.039056:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485875712, count: 4096 00000080:00200000:2.0:1788201707.039063:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485875712, bytes: 4096 00000080:00200000:2.0:1788201707.039064:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.039067:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.039068:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485875712, 485879808) 0 0 00000080:00200000:2.0:1788201707.039069:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.039073:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118622, 118622] 00020000:00200000:2.0:1788201707.039077:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485875712 chunk: [485875712, 485879808) 485879808, 4096 00020000:00200000:2.0:1788201707.039079:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.039090:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118622, 118622] 00000008:00010000:2.0:1788201707.039096:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.039098:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.039106:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039115:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485875712 485879807) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039124:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039132:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039140:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.039152:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485875712, 485879808) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.039156:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118622 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.039160:0:17463:0:(rw26.c:898:ll_write_end()) pos 485875712, len 4096, copied 4096 00000080:00200000:2.0:1788201707.039163:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004c039475[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118622] 00000080:00200000:2.0:1788201707.039165:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a7c6df19 fffffc0001029 2:0 ffff93c0c13f9b60 118622 lru 00000080:00200000:2.0:1788201707.039169:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f6c551bc 118622: 1< 2 - - > 2< 485875712 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70207 - -> 00000080:00200000:2.0:1788201707.039170:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004c039475 00000080:00200000:2.0:1788201707.039171:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.039172:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.039178:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.039186:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.039188:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.039189:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.039196:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.039198:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.039200:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.039202:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039207:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.039212:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.039213:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.039216:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.039218:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118622, 118622] 00000080:00200000:2.0:1788201707.039225:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.039227:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.039245:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.039247:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.039257:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485879808, count: 4096 00000080:00200000:2.0:1788201707.039263:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485879808, bytes: 4096 00000080:00200000:2.0:1788201707.039265:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.039267:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.039268:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485879808, 485883904) 0 0 00000080:00200000:2.0:1788201707.039270:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.039273:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118623, 118623] 00020000:00200000:2.0:1788201707.039275:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485879808 chunk: [485879808, 485883904) 485883904, 4096 00020000:00200000:2.0:1788201707.039276:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.039283:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118623, 118623] 00000008:00010000:2.0:1788201707.039286:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.039288:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.039293:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039299:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485879808 485883903) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039304:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039308:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039316:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.039320:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485879808, 485883904) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.039323:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118623 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.039327:0:17463:0:(rw26.c:898:ll_write_end()) pos 485879808, len 4096, copied 4096 00000080:00200000:2.0:1788201707.039329:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005dffe2d6[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118623] 00000080:00200000:2.0:1788201707.039330:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c6c775ab fffffc0001029 2:0 ffff93c0c13f9c40 118623 lru 00000080:00200000:2.0:1788201707.039334:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001da43b22 118623: 1< 2 - - > 2< 485879808 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70208 - -> 00000080:00200000:2.0:1788201707.039335:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005dffe2d6 00000080:00200000:2.0:1788201707.039336:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.039337:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.039343:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.039350:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.039352:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.039353:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.039359:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.039361:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.039362:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.039364:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039369:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.039378:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.039379:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.039382:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.039384:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118623, 118623] 00000080:00200000:2.0:1788201707.039390:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.039392:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.039409:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.039411:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.039421:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485883904, count: 4096 00000080:00200000:2.0:1788201707.039427:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485883904, bytes: 4096 00000080:00200000:2.0:1788201707.039428:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.039430:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.039432:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485883904, 485888000) 0 0 00000080:00200000:2.0:1788201707.039433:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.039436:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118624, 118624] 00020000:00200000:2.0:1788201707.039438:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485883904 chunk: [485883904, 485888000) 485888000, 4096 00020000:00200000:2.0:1788201707.039440:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.039446:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118624, 118624] 00000008:00010000:2.0:1788201707.039449:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.039451:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.039455:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039461:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485883904 485887999) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039466:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039475:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039479:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.039483:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485883904, 485888000) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.039486:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118624 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.039490:0:17463:0:(rw26.c:898:ll_write_end()) pos 485883904, len 4096, copied 4096 00000080:00200000:2.0:1788201707.039494:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000ba1c477[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118624] 00000080:00200000:2.0:1788201707.039496:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e4f8f203 fffffc0001029 2:0 ffff93c0c13f9540 118624 lru 00000080:00200000:2.0:1788201707.039502:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000439eb684 118624: 1< 2 - - > 2< 485883904 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70209 - -> 00000080:00200000:2.0:1788201707.039504:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000ba1c477 00000080:00200000:2.0:1788201707.039505:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.039507:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.039515:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.039527:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.039530:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.039532:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.039541:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.039543:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.039545:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.039547:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039556:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.039561:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.039562:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.039565:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.039567:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118624, 118624] 00000080:00200000:2.0:1788201707.039574:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.039576:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.039594:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.039596:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.039606:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485888000, count: 4096 00000080:00200000:2.0:1788201707.039612:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485888000, bytes: 4096 00000080:00200000:2.0:1788201707.039613:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.039615:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.039616:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485888000, 485892096) 0 0 00000080:00200000:2.0:1788201707.039618:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.039621:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118625, 118625] 00020000:00200000:2.0:1788201707.039623:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485888000 chunk: [485888000, 485892096) 485892096, 4096 00020000:00200000:2.0:1788201707.039625:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.039631:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118625, 118625] 00000008:00010000:2.0:1788201707.039634:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.039636:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.039641:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039647:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485888000 485892095) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039655:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039660:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039664:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.039668:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485888000, 485892096) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.039671:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118625 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.039675:0:17463:0:(rw26.c:898:ll_write_end()) pos 485888000, len 4096, copied 4096 00000080:00200000:2.0:1788201707.039677:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000943b0c73[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118625] 00000080:00200000:2.0:1788201707.039679:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000080abac4d fffffc0001029 2:0 ffff93c0c13f9460 118625 lru 00000080:00200000:2.0:1788201707.039683:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d7e12ca2 118625: 1< 2 - - > 2< 485888000 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70210 - -> 00000080:00200000:2.0:1788201707.039684:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000943b0c73 00000080:00200000:2.0:1788201707.039685:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.039686:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.039691:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.039699:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.039701:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.039702:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.039708:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.039710:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.039712:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.039714:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039719:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.039723:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.039725:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.039728:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.039729:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118625, 118625] 00000080:00200000:2.0:1788201707.039736:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.039737:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.039755:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.039757:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.039767:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485892096, count: 4096 00000080:00200000:2.0:1788201707.039772:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485892096, bytes: 4096 00000080:00200000:2.0:1788201707.039774:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.039776:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.039777:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485892096, 485896192) 0 0 00000080:00200000:2.0:1788201707.039779:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.039782:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118626, 118626] 00020000:00200000:2.0:1788201707.039784:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485892096 chunk: [485892096, 485896192) 485896192, 4096 00020000:00200000:2.0:1788201707.039785:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.039791:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118626, 118626] 00000008:00010000:2.0:1788201707.039795:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.039796:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.039801:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039810:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485892096 485896191) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039815:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039820:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039823:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.039827:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485892096, 485896192) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.039843:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118626 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.039847:0:17463:0:(rw26.c:898:ll_write_end()) pos 485892096, len 4096, copied 4096 00000080:00200000:2.0:1788201707.039849:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b0568e2e[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118626] 00000080:00200000:2.0:1788201707.039851:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a671214a fffffc0001029 2:0 ffff93c0c13f90e0 118626 lru 00000080:00200000:2.0:1788201707.039855:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ef148947 118626: 1< 2 - - > 2< 485892096 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70211 - -> 00000080:00200000:2.0:1788201707.039856:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b0568e2e 00000080:00200000:2.0:1788201707.039857:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.039858:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.039864:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.039871:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.039873:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.039874:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.039880:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.039882:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.039883:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.039886:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039891:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.039896:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.039897:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.039899:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.039901:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118626, 118626] 00000080:00200000:2.0:1788201707.039907:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.039909:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.039926:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.039929:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.039938:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485896192, count: 4096 00000080:00200000:2.0:1788201707.039944:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485896192, bytes: 4096 00000080:00200000:2.0:1788201707.039946:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.039948:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.039949:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485896192, 485900288) 0 0 00000080:00200000:2.0:1788201707.039951:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.039954:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118627, 118627] 00020000:00200000:2.0:1788201707.039956:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485896192 chunk: [485896192, 485900288) 485900288, 4096 00020000:00200000:2.0:1788201707.039957:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.039963:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118627, 118627] 00000008:00010000:2.0:1788201707.039966:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.039968:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.039973:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039979:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485896192 485900287) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039984:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039988:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.039992:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.039997:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485896192, 485900288) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.040000:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118627 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.040003:0:17463:0:(rw26.c:898:ll_write_end()) pos 485896192, len 4096, copied 4096 00000080:00200000:2.0:1788201707.040005:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000562b11c5[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118627] 00000080:00200000:2.0:1788201707.040007:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000d5b5a59 fffffc0001029 2:0 ffff93c0c13fa460 118627 lru 00000080:00200000:2.0:1788201707.040023:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000749e754b 118627: 1< 2 - - > 2< 485896192 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70212 - -> 00000080:00200000:2.0:1788201707.040024:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000562b11c5 00000080:00200000:2.0:1788201707.040025:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.040027:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.040033:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.040041:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.040043:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.040044:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.040050:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.040052:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.040053:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.040056:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040061:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.040066:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.040067:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.040070:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.040071:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118627, 118627] 00000080:00200000:2.0:1788201707.040078:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.040080:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.040097:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.040100:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.040109:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485900288, count: 4096 00000080:00200000:2.0:1788201707.040115:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485900288, bytes: 4096 00000080:00200000:2.0:1788201707.040117:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.040119:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.040120:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485900288, 485904384) 0 0 00000080:00200000:2.0:1788201707.040122:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.040125:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118628, 118628] 00020000:00200000:2.0:1788201707.040127:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485900288 chunk: [485900288, 485904384) 485904384, 4096 00020000:00200000:2.0:1788201707.040128:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.040135:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118628, 118628] 00000008:00010000:2.0:1788201707.040138:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.040139:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.040144:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040150:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485900288 485904383) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040155:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040159:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040163:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.040168:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485900288, 485904384) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.040171:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118628 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.040175:0:17463:0:(rw26.c:898:ll_write_end()) pos 485900288, len 4096, copied 4096 00000080:00200000:2.0:1788201707.040177:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006f1cbc6f[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118628] 00000080:00200000:2.0:1788201707.040179:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000025eb7c40 fffffc0001029 2:0 ffff93c0c13fa0e0 118628 lru 00000080:00200000:2.0:1788201707.040183:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008c4b50ea 118628: 1< 2 - - > 2< 485900288 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70213 - -> 00000080:00200000:2.0:1788201707.040184:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006f1cbc6f 00000080:00200000:2.0:1788201707.040185:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.040186:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.040191:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.040202:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.040204:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.040205:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.040211:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.040213:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.040214:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.040216:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040221:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.040226:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.040228:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.040230:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.040232:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118628, 118628] 00000080:00200000:2.0:1788201707.040238:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.040240:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.040257:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.040259:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.040269:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485904384, count: 4096 00000080:00200000:2.0:1788201707.040274:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485904384, bytes: 4096 00000080:00200000:2.0:1788201707.040276:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.040278:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.040279:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485904384, 485908480) 0 0 00000080:00200000:2.0:1788201707.040280:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.040284:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118629, 118629] 00020000:00200000:2.0:1788201707.040286:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485904384 chunk: [485904384, 485908480) 485908480, 4096 00020000:00200000:2.0:1788201707.040287:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.040293:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118629, 118629] 00000008:00010000:2.0:1788201707.040296:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.040298:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.040302:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040308:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485904384 485908479) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040313:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040318:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040322:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.040326:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485904384, 485908480) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.040329:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118629 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.040333:0:17463:0:(rw26.c:898:ll_write_end()) pos 485904384, len 4096, copied 4096 00000080:00200000:2.0:1788201707.040335:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b9f9bb08[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118629] 00000080:00200000:2.0:1788201707.040337:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a9fd3fda fffffc0001029 2:0 ffff93c0c13fa700 118629 lru 00000080:00200000:2.0:1788201707.040341:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000502193e0 118629: 1< 2 - - > 2< 485904384 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70214 - -> 00000080:00200000:2.0:1788201707.040342:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b9f9bb08 00000080:00200000:2.0:1788201707.040343:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.040344:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.040349:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.040356:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.040358:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.040359:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.040365:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.040367:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.040368:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.040370:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040375:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.040380:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.040381:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.040384:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.040386:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118629, 118629] 00000080:00200000:2.0:1788201707.040392:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.040393:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.040411:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.040413:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.040422:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485908480, count: 4096 00000080:00200000:2.0:1788201707.040428:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485908480, bytes: 4096 00000080:00200000:2.0:1788201707.040430:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.040432:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.040433:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485908480, 485912576) 0 0 00000080:00200000:2.0:1788201707.040435:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.040438:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118630, 118630] 00020000:00200000:2.0:1788201707.040440:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485908480 chunk: [485908480, 485912576) 485912576, 4096 00020000:00200000:2.0:1788201707.040441:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.040447:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118630, 118630] 00000008:00010000:2.0:1788201707.040450:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.040452:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.040457:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040463:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485908480 485912575) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040468:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040472:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040476:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.040481:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485908480, 485912576) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.040483:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118630 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.040487:0:17463:0:(rw26.c:898:ll_write_end()) pos 485908480, len 4096, copied 4096 00000080:00200000:2.0:1788201707.040491:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000057796bc[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118630] 00000080:00200000:2.0:1788201707.040494:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ee1dc3ed fffffc0001029 2:0 ffff93c0c13fa000 118630 lru 00000080:00200000:2.0:1788201707.040499:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001dcf7a53 118630: 1< 2 - - > 2< 485908480 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70215 - -> 00000080:00200000:2.0:1788201707.040503:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000057796bc 00000080:00200000:2.0:1788201707.040504:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.040506:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.040513:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.040525:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.040528:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.040531:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.040538:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.040540:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.040542:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.040544:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040549:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.040554:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.040556:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.040558:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.040560:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118630, 118630] 00000080:00200000:2.0:1788201707.040567:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.040569:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.040586:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.040588:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.040598:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485912576, count: 4096 00000080:00200000:2.0:1788201707.040604:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485912576, bytes: 4096 00000080:00200000:2.0:1788201707.040606:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.040608:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.040609:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485912576, 485916672) 0 0 00000080:00200000:2.0:1788201707.040611:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.040614:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118631, 118631] 00020000:00200000:2.0:1788201707.040616:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485912576 chunk: [485912576, 485916672) 485916672, 4096 00020000:00200000:2.0:1788201707.040617:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.040624:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118631, 118631] 00000008:00010000:2.0:1788201707.040627:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.040628:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.040633:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040639:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485912576 485916671) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040644:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040649:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040652:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.040656:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485912576, 485916672) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.040659:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118631 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.040663:0:17463:0:(rw26.c:898:ll_write_end()) pos 485912576, len 4096, copied 4096 00000080:00200000:2.0:1788201707.040665:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e6429758[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118631] 00000080:00200000:2.0:1788201707.040667:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000081aa9a15 fffffc0001029 2:0 ffff93c0c13faa80 118631 lru 00000080:00200000:2.0:1788201707.040671:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005576250c 118631: 1< 2 - - > 2< 485912576 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70216 - -> 00000080:00200000:2.0:1788201707.040672:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e6429758 00000080:00200000:2.0:1788201707.040673:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.040674:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.040679:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.040685:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.040687:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.040688:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.040694:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.040696:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.040697:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.040700:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040704:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.040709:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.040710:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.040713:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.040715:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118631, 118631] 00000080:00200000:2.0:1788201707.040721:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.040723:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.040740:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.040742:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.040752:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485916672, count: 4096 00000080:00200000:2.0:1788201707.040758:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485916672, bytes: 4096 00000080:00200000:2.0:1788201707.040759:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.040762:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.040763:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485916672, 485920768) 0 0 00000080:00200000:2.0:1788201707.040764:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.040767:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118632, 118632] 00020000:00200000:2.0:1788201707.040769:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485916672 chunk: [485916672, 485920768) 485920768, 4096 00020000:00200000:2.0:1788201707.040771:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.040777:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118632, 118632] 00000008:00010000:2.0:1788201707.040780:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.040782:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.040786:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040792:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485916672 485920767) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040798:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040802:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040806:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.040811:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485916672, 485920768) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.040813:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118632 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.040817:0:17463:0:(rw26.c:898:ll_write_end()) pos 485916672, len 4096, copied 4096 00000080:00200000:2.0:1788201707.040820:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f8a9d806[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118632] 00000080:00200000:2.0:1788201707.040821:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000081f2aab8 fffffc0001029 2:0 ffff93c0c13fa380 118632 lru 00000080:00200000:2.0:1788201707.040825:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c97a2956 118632: 1< 2 - - > 2< 485916672 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70217 - -> 00000080:00200000:2.0:1788201707.040826:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f8a9d806 00000080:00200000:2.0:1788201707.040827:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.040841:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.040847:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.040855:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.040856:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.040858:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.040864:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.040865:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.040867:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.040869:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040874:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.040879:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.040880:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.040882:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.040884:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118632, 118632] 00000080:00200000:2.0:1788201707.040890:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.040892:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.040909:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.040912:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.040921:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485920768, count: 4096 00000080:00200000:2.0:1788201707.040927:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485920768, bytes: 4096 00000080:00200000:2.0:1788201707.040929:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.040931:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.040932:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485920768, 485924864) 0 0 00000080:00200000:2.0:1788201707.040934:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.040937:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118633, 118633] 00020000:00200000:2.0:1788201707.040939:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485920768 chunk: [485920768, 485924864) 485924864, 4096 00020000:00200000:2.0:1788201707.040940:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.040946:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118633, 118633] 00000008:00010000:2.0:1788201707.040950:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.040951:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.040956:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040961:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485920768 485924863) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040967:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040971:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.040975:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.040979:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485920768, 485924864) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.040982:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118633 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.040986:0:17463:0:(rw26.c:898:ll_write_end()) pos 485920768, len 4096, copied 4096 00000080:00200000:2.0:1788201707.040988:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e19e9496[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118633] 00000080:00200000:2.0:1788201707.040990:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000002b83478 fffffc0001029 2:0 ffff93c0c13fa8c0 118633 lru 00000080:00200000:2.0:1788201707.040994:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d1bfed94 118633: 1< 2 - - > 2< 485920768 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70218 - -> 00000080:00200000:2.0:1788201707.040995:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e19e9496 00000080:00200000:2.0:1788201707.040996:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.040997:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.041002:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.041022:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.041024:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.041025:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.041032:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.041033:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.041035:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.041038:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041043:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.041047:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.041049:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.041051:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.041053:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118633, 118633] 00000080:00200000:2.0:1788201707.041060:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.041062:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.041080:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.041082:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.041092:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485924864, count: 4096 00000080:00200000:2.0:1788201707.041097:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485924864, bytes: 4096 00000080:00200000:2.0:1788201707.041099:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.041101:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.041103:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485924864, 485928960) 0 0 00000080:00200000:2.0:1788201707.041104:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.041107:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118634, 118634] 00020000:00200000:2.0:1788201707.041109:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485924864 chunk: [485924864, 485928960) 485928960, 4096 00020000:00200000:2.0:1788201707.041111:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.041117:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118634, 118634] 00000008:00010000:2.0:1788201707.041120:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.041121:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.041126:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041132:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485924864 485928959) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041138:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041142:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041146:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.041153:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485924864, 485928960) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.041156:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118634 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.041159:0:17463:0:(rw26.c:898:ll_write_end()) pos 485924864, len 4096, copied 4096 00000080:00200000:2.0:1788201707.041162:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004fc2e2fb[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118634] 00000080:00200000:2.0:1788201707.041164:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000092fcf57c fffffc0001029 2:0 ffff93c0c13faee0 118634 lru 00000080:00200000:2.0:1788201707.041167:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005d724183 118634: 1< 2 - - > 2< 485924864 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70219 - -> 00000080:00200000:2.0:1788201707.041169:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004fc2e2fb 00000080:00200000:2.0:1788201707.041169:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.041171:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.041175:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.041183:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.041184:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.041185:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.041191:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.041193:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.041194:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.041197:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041202:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.041207:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.041208:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.041211:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.041213:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118634, 118634] 00000080:00200000:2.0:1788201707.041219:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.041221:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.041238:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.041240:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.041250:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485928960, count: 4096 00000080:00200000:2.0:1788201707.041255:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485928960, bytes: 4096 00000080:00200000:2.0:1788201707.041257:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.041259:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.041260:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485928960, 485933056) 0 0 00000080:00200000:2.0:1788201707.041262:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.041265:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118635, 118635] 00020000:00200000:2.0:1788201707.041267:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485928960 chunk: [485928960, 485933056) 485933056, 4096 00020000:00200000:2.0:1788201707.041268:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.041274:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118635, 118635] 00000008:00010000:2.0:1788201707.041277:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.041279:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.041284:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041289:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485928960 485933055) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041294:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041299:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041306:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.041310:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485928960, 485933056) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.041313:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118635 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.041316:0:17463:0:(rw26.c:898:ll_write_end()) pos 485928960, len 4096, copied 4096 00000080:00200000:2.0:1788201707.041319:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005296ea8e[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118635] 00000080:00200000:2.0:1788201707.041320:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000a0130fd fffffc0001029 2:0 ffff93c0c13fa7e0 118635 lru 00000080:00200000:2.0:1788201707.041324:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000eaa821fb 118635: 1< 2 - - > 2< 485928960 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70220 - -> 00000080:00200000:2.0:1788201707.041325:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005296ea8e 00000080:00200000:2.0:1788201707.041326:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.041327:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.041332:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.041340:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.041341:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.041342:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.041348:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.041350:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.041351:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.041354:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041358:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.041366:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.041368:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.041370:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.041372:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118635, 118635] 00000080:00200000:2.0:1788201707.041378:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.041380:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.041397:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.041400:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.041409:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485933056, count: 4096 00000080:00200000:2.0:1788201707.041415:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485933056, bytes: 4096 00000080:00200000:2.0:1788201707.041416:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.041418:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.041419:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485933056, 485937152) 0 0 00000080:00200000:2.0:1788201707.041421:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.041424:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118636, 118636] 00020000:00200000:2.0:1788201707.041426:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485933056 chunk: [485933056, 485937152) 485937152, 4096 00020000:00200000:2.0:1788201707.041427:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.041434:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118636, 118636] 00000008:00010000:2.0:1788201707.041437:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.041438:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.041443:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041448:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485933056 485937151) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041454:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041462:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041466:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.041470:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485933056, 485937152) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.041473:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118636 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.041476:0:17463:0:(rw26.c:898:ll_write_end()) pos 485933056, len 4096, copied 4096 00000080:00200000:2.0:1788201707.041478:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000575db015[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118636] 00000080:00200000:2.0:1788201707.041480:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a194edcf fffffc0001029 2:0 ffff93c0c13fa2a0 118636 lru 00000080:00200000:2.0:1788201707.041483:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000022d0e64c 118636: 1< 2 - - > 2< 485933056 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70221 - -> 00000080:00200000:2.0:1788201707.041484:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000575db015 00000080:00200000:2.0:1788201707.041485:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.041486:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.041493:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.041505:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.041507:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.041510:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.041518:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.041522:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.041524:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.041528:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041539:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.041544:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.041546:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.041549:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.041551:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118636, 118636] 00000080:00200000:2.0:1788201707.041557:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.041559:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.041578:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.041580:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.041589:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485937152, count: 4096 00000080:00200000:2.0:1788201707.041595:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485937152, bytes: 4096 00000080:00200000:2.0:1788201707.041597:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.041599:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.041600:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485937152, 485941248) 0 0 00000080:00200000:2.0:1788201707.041602:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.041605:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118637, 118637] 00020000:00200000:2.0:1788201707.041607:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485937152 chunk: [485937152, 485941248) 485941248, 4096 00020000:00200000:2.0:1788201707.041608:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.041615:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118637, 118637] 00000008:00010000:2.0:1788201707.041618:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.041619:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.041625:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041631:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485937152 485941247) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041641:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041646:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041650:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.041654:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485937152, 485941248) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.041657:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118637 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.041661:0:17463:0:(rw26.c:898:ll_write_end()) pos 485937152, len 4096, copied 4096 00000080:00200000:2.0:1788201707.041663:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002e88288d[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118637] 00000080:00200000:2.0:1788201707.041664:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000023612695 fffffc0001029 2:0 ffff93c0c13fae00 118637 lru 00000080:00200000:2.0:1788201707.041668:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e1be0969 118637: 1< 2 - - > 2< 485937152 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70222 - -> 00000080:00200000:2.0:1788201707.041669:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002e88288d 00000080:00200000:2.0:1788201707.041670:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.041671:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.041676:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.041684:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.041685:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.041687:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.041693:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.041695:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.041697:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.041700:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041705:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.041710:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.041712:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.041715:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.041718:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118637, 118637] 00000080:00200000:2.0:1788201707.041724:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.041726:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.041746:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.041748:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.041758:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485941248, count: 4096 00000080:00200000:2.0:1788201707.041764:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485941248, bytes: 4096 00000080:00200000:2.0:1788201707.041765:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.041767:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.041768:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485941248, 485945344) 0 0 00000080:00200000:2.0:1788201707.041770:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.041773:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118638, 118638] 00020000:00200000:2.0:1788201707.041775:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485941248 chunk: [485941248, 485945344) 485945344, 4096 00020000:00200000:2.0:1788201707.041776:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.041782:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118638, 118638] 00000008:00010000:2.0:1788201707.041785:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.041787:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.041792:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041800:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485941248 485945343) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041806:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041810:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041814:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.041818:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485941248, 485945344) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.041821:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118638 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.041825:0:17463:0:(rw26.c:898:ll_write_end()) pos 485941248, len 4096, copied 4096 00000080:00200000:2.0:1788201707.041827:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009ac4d5e9[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118638] 00000080:00200000:2.0:1788201707.041842:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008d42ec7c fffffc0001029 2:0 ffff93c0c13fa1c0 118638 lru 00000080:00200000:2.0:1788201707.041845:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000adfbc570 118638: 1< 2 - - > 2< 485941248 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70223 - -> 00000080:00200000:2.0:1788201707.041846:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009ac4d5e9 00000080:00200000:2.0:1788201707.041847:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.041848:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.041854:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.041862:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.041864:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.041865:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.041871:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.041872:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.041874:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.041876:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041881:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.041886:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.041887:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.041890:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.041891:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118638, 118638] 00000080:00200000:2.0:1788201707.041898:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.041900:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.041917:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.041919:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.041929:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485945344, count: 4096 00000080:00200000:2.0:1788201707.041935:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485945344, bytes: 4096 00000080:00200000:2.0:1788201707.041936:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.041938:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.041939:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485945344, 485949440) 0 0 00000080:00200000:2.0:1788201707.041941:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.041944:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118639, 118639] 00020000:00200000:2.0:1788201707.041946:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485945344 chunk: [485945344, 485949440) 485949440, 4096 00020000:00200000:2.0:1788201707.041947:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.041954:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118639, 118639] 00000008:00010000:2.0:1788201707.041957:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.041958:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.041963:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041969:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485945344 485949439) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041974:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041978:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.041982:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.041986:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485945344, 485949440) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.041989:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118639 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.041992:0:17463:0:(rw26.c:898:ll_write_end()) pos 485945344, len 4096, copied 4096 00000080:00200000:2.0:1788201707.041995:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000dde21376[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118639] 00000080:00200000:2.0:1788201707.041996:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000644cfe80 fffffc0001029 2:0 ffff93c0c13fa620 118639 lru 00000080:00200000:2.0:1788201707.042000:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000dfddef5 118639: 1< 2 - - > 2< 485945344 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70224 - -> 00000080:00200000:2.0:1788201707.042001:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000dde21376 00000080:00200000:2.0:1788201707.042002:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.042003:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.042020:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.042028:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.042031:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.042032:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.042038:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.042040:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.042042:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.042044:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042049:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.042054:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.042055:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.042058:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.042060:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118639, 118639] 00000080:00200000:2.0:1788201707.042066:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.042068:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.042086:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.042088:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.042098:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485949440, count: 4096 00000080:00200000:2.0:1788201707.042103:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485949440, bytes: 4096 00000080:00200000:2.0:1788201707.042105:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.042107:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.042108:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485949440, 485953536) 0 0 00000080:00200000:2.0:1788201707.042110:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.042113:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118640, 118640] 00020000:00200000:2.0:1788201707.042115:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485949440 chunk: [485949440, 485953536) 485953536, 4096 00020000:00200000:2.0:1788201707.042116:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.042123:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118640, 118640] 00000008:00010000:2.0:1788201707.042126:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.042127:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.042132:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042138:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485949440 485953535) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042143:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042147:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042151:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.042155:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485949440, 485953536) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.042158:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118640 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.042162:0:17463:0:(rw26.c:898:ll_write_end()) pos 485949440, len 4096, copied 4096 00000080:00200000:2.0:1788201707.042164:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000014309568[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118640] 00000080:00200000:2.0:1788201707.042166:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cd3e3269 fffffc0001029 2:0 ffff93c0c13fad20 118640 lru 00000080:00200000:2.0:1788201707.042169:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000493965d1 118640: 1< 2 - - > 2< 485949440 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70225 - -> 00000080:00200000:2.0:1788201707.042170:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000014309568 00000080:00200000:2.0:1788201707.042171:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.042172:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.042177:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.042188:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.042190:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.042191:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.042196:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.042198:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.042200:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.042202:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042207:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.042212:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.042213:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.042216:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.042218:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118640, 118640] 00000080:00200000:2.0:1788201707.042224:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.042226:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.042243:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.042246:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.042255:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485953536, count: 4096 00000080:00200000:2.0:1788201707.042261:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485953536, bytes: 4096 00000080:00200000:2.0:1788201707.042263:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.042265:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.042266:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485953536, 485957632) 0 0 00000080:00200000:2.0:1788201707.042268:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.042271:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118641, 118641] 00020000:00200000:2.0:1788201707.042273:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485953536 chunk: [485953536, 485957632) 485957632, 4096 00020000:00200000:2.0:1788201707.042274:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.042281:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118641, 118641] 00000008:00010000:2.0:1788201707.042284:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.042285:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.042290:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042296:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485953536 485957631) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042301:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042305:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042309:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.042313:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485953536, 485957632) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.042316:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118641 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.042320:0:17463:0:(rw26.c:898:ll_write_end()) pos 485953536, len 4096, copied 4096 00000080:00200000:2.0:1788201707.042322:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000873f8aba[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118641] 00000080:00200000:2.0:1788201707.042323:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000807d0075 fffffc0001029 2:0 ffff93c0c13fa9a0 118641 lru 00000080:00200000:2.0:1788201707.042327:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f22bca3f 118641: 1< 2 - - > 2< 485953536 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70226 - -> 00000080:00200000:2.0:1788201707.042328:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000873f8aba 00000080:00200000:2.0:1788201707.042328:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.042330:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.042335:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.042343:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.042344:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.042346:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.042351:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.042353:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.042354:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.042357:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042361:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.042365:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.042367:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.042369:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.042371:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118641, 118641] 00000080:00200000:2.0:1788201707.042377:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.042379:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.042396:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.042398:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.042408:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485957632, count: 4096 00000080:00200000:2.0:1788201707.042414:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485957632, bytes: 4096 00000080:00200000:2.0:1788201707.042415:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.042417:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.042418:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485957632, 485961728) 0 0 00000080:00200000:2.0:1788201707.042420:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.042423:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118642, 118642] 00020000:00200000:2.0:1788201707.042425:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485957632 chunk: [485957632, 485961728) 485961728, 4096 00020000:00200000:2.0:1788201707.042427:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.042433:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118642, 118642] 00000008:00010000:2.0:1788201707.042436:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.042437:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.042442:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042448:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485957632 485961727) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042453:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042457:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042461:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.042465:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485957632, 485961728) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.042468:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118642 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.042471:0:17463:0:(rw26.c:898:ll_write_end()) pos 485957632, len 4096, copied 4096 00000080:00200000:2.0:1788201707.042473:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000eab9cbe8[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118642] 00000080:00200000:2.0:1788201707.042475:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fad70c09 fffffc0001029 2:0 ffff93c0c13fab60 118642 lru 00000080:00200000:2.0:1788201707.042479:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000186960bf 118642: 1< 2 - - > 2< 485957632 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70227 - -> 00000080:00200000:2.0:1788201707.042481:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000eab9cbe8 00000080:00200000:2.0:1788201707.042482:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.042483:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.042489:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.042500:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.042503:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.042505:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.042514:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.042517:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.042519:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.042523:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042530:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.042538:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.042541:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.042545:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.042548:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118642, 118642] 00000080:00200000:2.0:1788201707.042558:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.042560:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.042578:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.042581:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.042591:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485961728, count: 4096 00000080:00200000:2.0:1788201707.042596:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485961728, bytes: 4096 00000080:00200000:2.0:1788201707.042598:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.042601:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.042602:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485961728, 485965824) 0 0 00000080:00200000:2.0:1788201707.042603:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.042607:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118643, 118643] 00020000:00200000:2.0:1788201707.042609:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485961728 chunk: [485961728, 485965824) 485965824, 4096 00020000:00200000:2.0:1788201707.042610:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.042616:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118643, 118643] 00000008:00010000:2.0:1788201707.042620:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.042621:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.042626:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042632:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485961728 485965823) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042637:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042642:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042646:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.042650:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485961728, 485965824) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.042653:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118643 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.042657:0:17463:0:(rw26.c:898:ll_write_end()) pos 485961728, len 4096, copied 4096 00000080:00200000:2.0:1788201707.042659:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000da9439ab[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118643] 00000080:00200000:2.0:1788201707.042661:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000021d88778 fffffc0001029 2:0 ffff93c0c13fac40 118643 lru 00000080:00200000:2.0:1788201707.042665:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000000cedcb8 118643: 1< 2 - - > 2< 485961728 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70228 - -> 00000080:00200000:2.0:1788201707.042666:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000da9439ab 00000080:00200000:2.0:1788201707.042667:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.042668:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.042673:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.042680:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.042682:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.042683:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.042689:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.042691:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.042692:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.042695:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042699:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.042704:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.042705:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.042708:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.042710:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118643, 118643] 00000080:00200000:2.0:1788201707.042716:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.042718:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.042735:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.042738:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.042747:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485965824, count: 4096 00000080:00200000:2.0:1788201707.042753:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485965824, bytes: 4096 00000080:00200000:2.0:1788201707.042755:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.042757:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.042758:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485965824, 485969920) 0 0 00000080:00200000:2.0:1788201707.042759:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.042762:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118644, 118644] 00020000:00200000:2.0:1788201707.042764:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485965824 chunk: [485965824, 485969920) 485969920, 4096 00020000:00200000:2.0:1788201707.042766:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.042772:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118644, 118644] 00000008:00010000:2.0:1788201707.042775:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.042776:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.042781:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042786:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485965824 485969919) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042792:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042797:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042800:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.042805:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485965824, 485969920) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.042808:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118644 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.042812:0:17463:0:(rw26.c:898:ll_write_end()) pos 485965824, len 4096, copied 4096 00000080:00200000:2.0:1788201707.042814:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008dd1aab7[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118644] 00000080:00200000:2.0:1788201707.042816:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000c478a42 fffffc0001029 2:0 ffff93c0c13fa540 118644 lru 00000080:00200000:2.0:1788201707.042820:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b1748837 118644: 1< 2 - - > 2< 485965824 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70229 - -> 00000080:00200000:2.0:1788201707.042821:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008dd1aab7 00000080:00200000:2.0:1788201707.042822:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.042823:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.042841:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.042850:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.042852:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.042853:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.042859:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.042860:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.042862:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.042864:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042869:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.042874:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.042875:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.042878:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.042879:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118644, 118644] 00000080:00200000:2.0:1788201707.042886:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.042888:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.042905:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.042907:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.042917:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485969920, count: 4096 00000080:00200000:2.0:1788201707.042922:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485969920, bytes: 4096 00000080:00200000:2.0:1788201707.042924:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.042926:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.042927:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485969920, 485974016) 0 0 00000080:00200000:2.0:1788201707.042929:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.042932:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118645, 118645] 00020000:00200000:2.0:1788201707.042934:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485969920 chunk: [485969920, 485974016) 485974016, 4096 00020000:00200000:2.0:1788201707.042936:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.042942:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118645, 118645] 00000008:00010000:2.0:1788201707.042945:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.042947:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.042952:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042957:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485969920 485974015) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042963:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042967:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.042971:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.042975:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485969920, 485974016) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.042978:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118645 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.042982:0:17463:0:(rw26.c:898:ll_write_end()) pos 485969920, len 4096, copied 4096 00000080:00200000:2.0:1788201707.042984:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000055606311[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118645] 00000080:00200000:2.0:1788201707.042986:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005bdb11a3 fffffc0001029 2:0 ffff93c0c13fb1c0 118645 lru 00000080:00200000:2.0:1788201707.042989:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d438f341 118645: 1< 2 - - > 2< 485969920 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70230 - -> 00000080:00200000:2.0:1788201707.042991:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000055606311 00000080:00200000:2.0:1788201707.042991:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.042993:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.042998:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.043005:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.043020:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.043022:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.043029:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.043031:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.043033:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.043035:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043040:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.043045:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.043047:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.043049:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.043051:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118645, 118645] 00000080:00200000:2.0:1788201707.043058:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.043060:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.043078:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.043080:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.043090:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485974016, count: 4096 00000080:00200000:2.0:1788201707.043095:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485974016, bytes: 4096 00000080:00200000:2.0:1788201707.043097:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.043099:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.043100:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485974016, 485978112) 0 0 00000080:00200000:2.0:1788201707.043102:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.043105:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118646, 118646] 00020000:00200000:2.0:1788201707.043107:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485974016 chunk: [485974016, 485978112) 485978112, 4096 00020000:00200000:2.0:1788201707.043108:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.043115:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118646, 118646] 00000008:00010000:2.0:1788201707.043118:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.043119:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.043124:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043130:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485974016 485978111) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043135:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043139:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043143:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.043151:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485974016, 485978112) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.043154:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118646 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.043157:0:17463:0:(rw26.c:898:ll_write_end()) pos 485974016, len 4096, copied 4096 00000080:00200000:2.0:1788201707.043160:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000058bd01b[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118646] 00000080:00200000:2.0:1788201707.043162:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b87f4974 fffffc0001029 2:0 ffff93c0c13fb620 118646 lru 00000080:00200000:2.0:1788201707.043165:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c6f26f49 118646: 1< 2 - - > 2< 485974016 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70231 - -> 00000080:00200000:2.0:1788201707.043167:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000058bd01b 00000080:00200000:2.0:1788201707.043167:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.043169:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.043174:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.043181:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.043183:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.043184:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.043190:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.043191:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.043193:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.043195:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043200:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.043205:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.043206:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.043209:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.043211:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118646, 118646] 00000080:00200000:2.0:1788201707.043217:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.043219:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.043236:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.043238:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.043248:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485978112, count: 4096 00000080:00200000:2.0:1788201707.043254:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485978112, bytes: 4096 00000080:00200000:2.0:1788201707.043255:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.043257:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.043258:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485978112, 485982208) 0 0 00000080:00200000:2.0:1788201707.043260:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.043263:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118647, 118647] 00020000:00200000:2.0:1788201707.043265:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485978112 chunk: [485978112, 485982208) 485982208, 4096 00020000:00200000:2.0:1788201707.043266:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.043272:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118647, 118647] 00000008:00010000:2.0:1788201707.043276:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.043277:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.043282:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043287:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485978112 485982207) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043293:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043297:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043304:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.043309:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485978112, 485982208) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.043311:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118647 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.043315:0:17463:0:(rw26.c:898:ll_write_end()) pos 485978112, len 4096, copied 4096 00000080:00200000:2.0:1788201707.043317:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000418b7e1[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118647] 00000080:00200000:2.0:1788201707.043318:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007277e19b fffffc0001029 2:0 ffff93c0c13fbd20 118647 lru 00000080:00200000:2.0:1788201707.043322:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c395028e 118647: 1< 2 - - > 2< 485978112 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70232 - -> 00000080:00200000:2.0:1788201707.043323:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000418b7e1 00000080:00200000:2.0:1788201707.043324:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.043325:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.043330:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.043338:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.043339:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.043341:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.043346:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.043348:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.043349:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.043352:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043356:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.043364:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.043366:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.043368:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.043370:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118647, 118647] 00000080:00200000:2.0:1788201707.043376:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.043378:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.043395:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.043397:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.043407:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485982208, count: 4096 00000080:00200000:2.0:1788201707.043413:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485982208, bytes: 4096 00000080:00200000:2.0:1788201707.043414:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.043416:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.043417:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485982208, 485986304) 0 0 00000080:00200000:2.0:1788201707.043419:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.043422:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118648, 118648] 00020000:00200000:2.0:1788201707.043424:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485982208 chunk: [485982208, 485986304) 485986304, 4096 00020000:00200000:2.0:1788201707.043425:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.043432:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118648, 118648] 00000008:00010000:2.0:1788201707.043435:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.043436:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.043441:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043446:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485982208 485986303) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043452:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043460:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043464:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.043468:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485982208, 485986304) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.043471:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118648 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.043474:0:17463:0:(rw26.c:898:ll_write_end()) pos 485982208, len 4096, copied 4096 00000080:00200000:2.0:1788201707.043476:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000819008fd[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118648] 00000080:00200000:2.0:1788201707.043478:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000be4f5c41 fffffc0001029 2:0 ffff93c0c13fb9a0 118648 lru 00000080:00200000:2.0:1788201707.043481:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000096051964 118648: 1< 2 - - > 2< 485982208 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70233 - -> 00000080:00200000:2.0:1788201707.043482:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000819008fd 00000080:00200000:2.0:1788201707.043483:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.043484:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.043490:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.043501:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.043504:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.043506:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.043515:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.043518:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.043520:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.043524:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043537:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.043545:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.043547:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.043551:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.043553:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118648, 118648] 00000080:00200000:2.0:1788201707.043560:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.043562:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.043581:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.043583:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.043593:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485986304, count: 4096 00000080:00200000:2.0:1788201707.043599:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485986304, bytes: 4096 00000080:00200000:2.0:1788201707.043601:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.043603:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.043604:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485986304, 485990400) 0 0 00000080:00200000:2.0:1788201707.043606:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.043609:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118649, 118649] 00020000:00200000:2.0:1788201707.043611:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485986304 chunk: [485986304, 485990400) 485990400, 4096 00020000:00200000:2.0:1788201707.043612:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.043619:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118649, 118649] 00000008:00010000:2.0:1788201707.043622:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.043624:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.043629:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043635:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485986304 485990399) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043643:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043648:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043652:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.043656:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485986304, 485990400) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.043659:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118649 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.043663:0:17463:0:(rw26.c:898:ll_write_end()) pos 485986304, len 4096, copied 4096 00000080:00200000:2.0:1788201707.043666:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000053a6476[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118649] 00000080:00200000:2.0:1788201707.043668:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001ca25228 fffffc0001029 2:0 ffff93c0c13fbb60 118649 lru 00000080:00200000:2.0:1788201707.043671:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002ca09a1e 118649: 1< 2 - - > 2< 485986304 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70234 - -> 00000080:00200000:2.0:1788201707.043673:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000053a6476 00000080:00200000:2.0:1788201707.043674:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.043675:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.043680:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.043688:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.043689:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.043690:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.043696:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.043698:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.043700:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.043702:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043707:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.043711:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.043713:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.043715:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.043717:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118649, 118649] 00000080:00200000:2.0:1788201707.043724:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.043726:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.043743:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.043745:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.043755:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485990400, count: 4096 00000080:00200000:2.0:1788201707.043760:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485990400, bytes: 4096 00000080:00200000:2.0:1788201707.043762:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.043764:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.043765:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485990400, 485994496) 0 0 00000080:00200000:2.0:1788201707.043767:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.043770:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118650, 118650] 00020000:00200000:2.0:1788201707.043771:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485990400 chunk: [485990400, 485994496) 485994496, 4096 00020000:00200000:2.0:1788201707.043773:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.043779:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118650, 118650] 00000008:00010000:2.0:1788201707.043782:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.043784:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.043788:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043797:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485990400 485994495) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043802:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043806:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043810:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.043815:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485990400, 485994496) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.043818:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118650 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.043821:0:17463:0:(rw26.c:898:ll_write_end()) pos 485990400, len 4096, copied 4096 00000080:00200000:2.0:1788201707.043824:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000098d787[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118650] 00000080:00200000:2.0:1788201707.043825:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004312f85f fffffc0001029 2:0 ffff93c0c13fbc40 118650 lru 00000080:00200000:2.0:1788201707.043842:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004b817cad 118650: 1< 2 - - > 2< 485990400 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70235 - -> 00000080:00200000:2.0:1788201707.043844:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000098d787 00000080:00200000:2.0:1788201707.043844:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.043846:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.043852:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.043859:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.043861:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.043863:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.043868:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.043870:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.043872:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.043874:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043879:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.043884:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.043885:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.043887:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.043889:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118650, 118650] 00000080:00200000:2.0:1788201707.043896:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.043898:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.043915:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.043917:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.043927:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485994496, count: 4096 00000080:00200000:2.0:1788201707.043932:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485994496, bytes: 4096 00000080:00200000:2.0:1788201707.043934:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.043936:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.043937:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485994496, 485998592) 0 0 00000080:00200000:2.0:1788201707.043939:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.043942:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118651, 118651] 00020000:00200000:2.0:1788201707.043944:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485994496 chunk: [485994496, 485998592) 485998592, 4096 00020000:00200000:2.0:1788201707.043945:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.043951:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118651, 118651] 00000008:00010000:2.0:1788201707.043954:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.043956:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.043961:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043967:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485994496 485998591) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043972:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043977:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.043980:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.043984:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485994496, 485998592) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.043987:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118651 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.043991:0:17463:0:(rw26.c:898:ll_write_end()) pos 485994496, len 4096, copied 4096 00000080:00200000:2.0:1788201707.043993:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000092d85bab[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118651] 00000080:00200000:2.0:1788201707.043995:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008fbdf551 fffffc0001029 2:0 ffff93c0c13fb540 118651 lru 00000080:00200000:2.0:1788201707.043998:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000dd7ffdf7 118651: 1< 2 - - > 2< 485994496 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70236 - -> 00000080:00200000:2.0:1788201707.044000:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000092d85bab 00000080:00200000:2.0:1788201707.044000:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.044001:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.044006:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.044029:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.044031:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.044032:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.044039:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.044040:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.044042:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.044045:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044050:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.044055:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.044056:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.044058:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.044060:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118651, 118651] 00000080:00200000:2.0:1788201707.044067:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.044069:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.044086:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.044089:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.044098:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 485998592, count: 4096 00000080:00200000:2.0:1788201707.044104:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 485998592, bytes: 4096 00000080:00200000:2.0:1788201707.044106:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.044108:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.044109:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [485998592, 486002688) 0 0 00000080:00200000:2.0:1788201707.044111:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.044114:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118652, 118652] 00020000:00200000:2.0:1788201707.044116:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 485998592 chunk: [485998592, 486002688) 486002688, 4096 00020000:00200000:2.0:1788201707.044117:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.044124:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118652, 118652] 00000008:00010000:2.0:1788201707.044127:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.044129:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.044133:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044139:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (485998592 486002687) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044144:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044149:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044152:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.044156:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [485998592, 486002688) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.044159:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118652 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.044163:0:17463:0:(rw26.c:898:ll_write_end()) pos 485998592, len 4096, copied 4096 00000080:00200000:2.0:1788201707.044165:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005f8e08b1[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118652] 00000080:00200000:2.0:1788201707.044167:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000009b99f6c fffffc0001029 2:0 ffff93c0c13fb460 118652 lru 00000080:00200000:2.0:1788201707.044170:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000055f4ede6 118652: 1< 2 - - > 2< 485998592 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70237 - -> 00000080:00200000:2.0:1788201707.044172:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005f8e08b1 00000080:00200000:2.0:1788201707.044172:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.044174:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.044179:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.044189:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.044191:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.044192:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.044198:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.044200:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.044201:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.044204:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044208:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.044213:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.044214:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.044217:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.044219:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118652, 118652] 00000080:00200000:2.0:1788201707.044225:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.044227:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.044244:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.044246:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.044255:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486002688, count: 4096 00000080:00200000:2.0:1788201707.044261:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486002688, bytes: 4096 00000080:00200000:2.0:1788201707.044263:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.044265:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.044266:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486002688, 486006784) 0 0 00000080:00200000:2.0:1788201707.044268:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.044271:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118653, 118653] 00020000:00200000:2.0:1788201707.044273:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486002688 chunk: [486002688, 486006784) 486006784, 4096 00020000:00200000:2.0:1788201707.044274:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.044281:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118653, 118653] 00000008:00010000:2.0:1788201707.044284:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.044285:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.044290:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044296:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486002688 486006783) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044301:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044305:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044309:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.044314:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486002688, 486006784) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.044317:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118653 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.044321:0:17463:0:(rw26.c:898:ll_write_end()) pos 486002688, len 4096, copied 4096 00000080:00200000:2.0:1788201707.044323:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c94c1c76[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118653] 00000080:00200000:2.0:1788201707.044325:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000083fb8310 fffffc0001029 2:0 ffff93c0c13fb0e0 118653 lru 00000080:00200000:2.0:1788201707.044328:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000005bca0d 118653: 1< 2 - - > 2< 486002688 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70238 - -> 00000080:00200000:2.0:1788201707.044330:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c94c1c76 00000080:00200000:2.0:1788201707.044331:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.044332:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.044337:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.044345:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.044346:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.044348:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.044353:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.044355:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.044356:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.044359:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044363:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.044368:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.044369:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.044372:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.044374:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118653, 118653] 00000080:00200000:2.0:1788201707.044380:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.044382:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.044399:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.044401:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.044411:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486006784, count: 4096 00000080:00200000:2.0:1788201707.044416:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486006784, bytes: 4096 00000080:00200000:2.0:1788201707.044418:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.044420:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.044421:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486006784, 486010880) 0 0 00000080:00200000:2.0:1788201707.044423:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.044426:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118654, 118654] 00020000:00200000:2.0:1788201707.044428:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486006784 chunk: [486006784, 486010880) 486010880, 4096 00020000:00200000:2.0:1788201707.044429:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.044436:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118654, 118654] 00000008:00010000:2.0:1788201707.044439:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.044440:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.044445:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044451:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486006784 486010879) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044456:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044461:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044464:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.044468:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486006784, 486010880) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.044471:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118654 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.044474:0:17463:0:(rw26.c:898:ll_write_end()) pos 486006784, len 4096, copied 4096 00000080:00200000:2.0:1788201707.044477:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f0d4bbf5[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118654] 00000080:00200000:2.0:1788201707.044478:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002136ae5a fffffc0001029 2:0 ffff93c0c13fb700 118654 lru 00000080:00200000:2.0:1788201707.044482:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000029b0c4dd 118654: 1< 2 - - > 2< 486006784 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70239 - -> 00000080:00200000:2.0:1788201707.044484:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f0d4bbf5 00000080:00200000:2.0:1788201707.044485:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.044486:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.044493:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.044505:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.044507:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.044509:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.044518:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.044521:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.044523:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.044527:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044535:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.044542:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.044544:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.044548:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.044550:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118654, 118654] 00000080:00200000:2.0:1788201707.044557:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.044559:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.044577:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.044579:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.044590:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486010880, count: 4096 00000080:00200000:2.0:1788201707.044595:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486010880, bytes: 4096 00000080:00200000:2.0:1788201707.044597:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.044599:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.044601:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486010880, 486014976) 0 0 00000080:00200000:2.0:1788201707.044602:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.044605:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118655, 118655] 00020000:00200000:2.0:1788201707.044607:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486010880 chunk: [486010880, 486014976) 486014976, 4096 00020000:00200000:2.0:1788201707.044609:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.044615:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118655, 118655] 00000008:00010000:2.0:1788201707.044619:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.044620:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.044625:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044631:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486010880 486014975) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044636:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044641:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044645:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.044649:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486010880, 486014976) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.044652:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118655 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.044656:0:17463:0:(rw26.c:898:ll_write_end()) pos 486010880, len 4096, copied 4096 00000080:00200000:2.0:1788201707.044658:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003a99b8a7[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118655] 00000080:00200000:2.0:1788201707.044660:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002c8deaed fffffc0001029 2:0 ffff93c0c13fb000 118655 lru 00000080:00200000:2.0:1788201707.044663:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005b79d1a2 118655: 1< 2 - - > 2< 486010880 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70240 - -> 00000080:00200000:2.0:1788201707.044665:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003a99b8a7 00000080:00200000:2.0:1788201707.044666:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.044667:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.044672:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.044679:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.044681:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.044682:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.044688:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.044690:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.044691:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.044694:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044698:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.044703:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.044704:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.044707:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.044709:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118655, 118655] 00000080:00200000:2.0:1788201707.044715:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.044717:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.044734:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.044736:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.044746:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486014976, count: 4096 00000080:00200000:2.0:1788201707.044752:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486014976, bytes: 4096 00000080:00200000:2.0:1788201707.044754:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.044756:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.044757:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486014976, 486019072) 0 0 00000080:00200000:2.0:1788201707.044758:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.044761:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118656, 118656] 00020000:00200000:2.0:1788201707.044763:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486014976 chunk: [486014976, 486019072) 486019072, 4096 00020000:00200000:2.0:1788201707.044765:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.044771:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118656, 118656] 00000008:00010000:2.0:1788201707.044775:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.044776:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.044781:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044786:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486014976 486019071) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044792:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044796:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044800:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.044804:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486014976, 486019072) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.044807:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118656 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.044811:0:17463:0:(rw26.c:898:ll_write_end()) pos 486014976, len 4096, copied 4096 00000080:00200000:2.0:1788201707.044813:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b8558210[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118656] 00000080:00200000:2.0:1788201707.044815:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000adbaa27d fffffc0001029 2:0 ffff93c0c13fba80 118656 lru 00000080:00200000:2.0:1788201707.044818:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000064c7985e 118656: 1< 2 - - > 2< 486014976 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70241 - -> 00000080:00200000:2.0:1788201707.044819:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b8558210 00000080:00200000:2.0:1788201707.044820:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.044821:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.044827:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.044847:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.044850:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.044851:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.044856:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.044858:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.044860:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.044862:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044867:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.044872:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.044873:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.044876:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.044877:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118656, 118656] 00000080:00200000:2.0:1788201707.044884:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.044886:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.044904:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.044906:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.044916:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486019072, count: 4096 00000080:00200000:2.0:1788201707.044921:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486019072, bytes: 4096 00000080:00200000:2.0:1788201707.044923:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.044925:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.044926:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486019072, 486023168) 0 0 00000080:00200000:2.0:1788201707.044928:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.044931:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118657, 118657] 00020000:00200000:2.0:1788201707.044933:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486019072 chunk: [486019072, 486023168) 486023168, 4096 00020000:00200000:2.0:1788201707.044934:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.044941:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118657, 118657] 00000008:00010000:2.0:1788201707.044944:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.044945:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.044950:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044956:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486019072 486023167) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044961:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044966:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.044969:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.044973:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486019072, 486023168) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.044977:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118657 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.044980:0:17463:0:(rw26.c:898:ll_write_end()) pos 486019072, len 4096, copied 4096 00000080:00200000:2.0:1788201707.044982:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000186035e5[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118657] 00000080:00200000:2.0:1788201707.044984:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002df5f58d fffffc0001029 2:0 ffff93c0c13fb380 118657 lru 00000080:00200000:2.0:1788201707.044988:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000050f975b2 118657: 1< 2 - - > 2< 486019072 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70242 - -> 00000080:00200000:2.0:1788201707.044989:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000186035e5 00000080:00200000:2.0:1788201707.044990:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.044991:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.044997:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.045004:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.045005:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.045007:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.045025:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.045027:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.045028:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.045031:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045036:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.045041:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.045042:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.045045:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.045047:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118657, 118657] 00000080:00200000:2.0:1788201707.045053:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.045055:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.045073:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.045075:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.045085:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486023168, count: 4096 00000080:00200000:2.0:1788201707.045091:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486023168, bytes: 4096 00000080:00200000:2.0:1788201707.045092:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.045095:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.045096:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486023168, 486027264) 0 0 00000080:00200000:2.0:1788201707.045097:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.045100:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118658, 118658] 00020000:00200000:2.0:1788201707.045102:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486023168 chunk: [486023168, 486027264) 486027264, 4096 00020000:00200000:2.0:1788201707.045104:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.045110:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118658, 118658] 00000008:00010000:2.0:1788201707.045113:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.045115:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.045120:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045126:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486023168 486027263) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045131:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045135:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045139:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.045146:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486023168, 486027264) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.045149:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118658 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.045153:0:17463:0:(rw26.c:898:ll_write_end()) pos 486023168, len 4096, copied 4096 00000080:00200000:2.0:1788201707.045155:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f00840e3[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118658] 00000080:00200000:2.0:1788201707.045157:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003342ef37 fffffc0001029 2:0 ffff93c0c13fb8c0 118658 lru 00000080:00200000:2.0:1788201707.045161:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000072f7517e 118658: 1< 2 - - > 2< 486023168 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70243 - -> 00000080:00200000:2.0:1788201707.045162:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f00840e3 00000080:00200000:2.0:1788201707.045163:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.045164:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.045169:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.045177:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.045178:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.045179:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.045185:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.045187:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.045189:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.045191:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045196:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.045200:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.045201:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.045204:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.045206:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118658, 118658] 00000080:00200000:2.0:1788201707.045212:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.045214:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.045231:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.045233:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.045243:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486027264, count: 4096 00000080:00200000:2.0:1788201707.045248:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486027264, bytes: 4096 00000080:00200000:2.0:1788201707.045250:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.045252:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.045253:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486027264, 486031360) 0 0 00000080:00200000:2.0:1788201707.045255:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.045258:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118659, 118659] 00020000:00200000:2.0:1788201707.045260:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486027264 chunk: [486027264, 486031360) 486031360, 4096 00020000:00200000:2.0:1788201707.045261:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.045268:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118659, 118659] 00000008:00010000:2.0:1788201707.045271:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.045272:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.045277:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045282:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486027264 486031359) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045287:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045292:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045299:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.045304:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486027264, 486031360) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.045306:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118659 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.045310:0:17463:0:(rw26.c:898:ll_write_end()) pos 486027264, len 4096, copied 4096 00000080:00200000:2.0:1788201707.045312:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000be25a650[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118659] 00000080:00200000:2.0:1788201707.045314:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a301833f fffffc0001029 2:0 ffff93c0c13417e0 118659 lru 00000080:00200000:2.0:1788201707.045317:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a959be09 118659: 1< 2 - - > 2< 486027264 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70244 - -> 00000080:00200000:2.0:1788201707.045319:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000be25a650 00000080:00200000:2.0:1788201707.045319:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.045320:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.045326:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.045333:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.045334:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.045336:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.045341:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.045343:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.045344:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.045347:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045351:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.045360:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.045361:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.045364:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.045366:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118659, 118659] 00000080:00200000:2.0:1788201707.045372:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.045374:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.045391:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.045393:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.045402:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486031360, count: 4096 00000080:00200000:2.0:1788201707.045408:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486031360, bytes: 4096 00000080:00200000:2.0:1788201707.045410:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.045412:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.045413:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486031360, 486035456) 0 0 00000080:00200000:2.0:1788201707.045414:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.045417:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118660, 118660] 00020000:00200000:2.0:1788201707.045419:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486031360 chunk: [486031360, 486035456) 486035456, 4096 00020000:00200000:2.0:1788201707.045421:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.045427:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118660, 118660] 00000008:00010000:2.0:1788201707.045430:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.045432:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.045436:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045442:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486031360 486035455) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045448:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045456:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045460:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.045464:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486031360, 486035456) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.045467:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118660 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.045470:0:17463:0:(rw26.c:898:ll_write_end()) pos 486031360, len 4096, copied 4096 00000080:00200000:2.0:1788201707.045472:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000019f48d4d[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118660] 00000080:00200000:2.0:1788201707.045474:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000397686b1 fffffc0001029 2:0 ffff93c0c13412a0 118660 lru 00000080:00200000:2.0:1788201707.045477:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008f08b764 118660: 1< 2 - - > 2< 486031360 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70245 - -> 00000080:00200000:2.0:1788201707.045478:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000019f48d4d 00000080:00200000:2.0:1788201707.045479:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.045480:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.045485:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.045496:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.045499:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.045501:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.045509:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.045512:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.045514:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.045518:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045531:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.045539:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.045541:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.045545:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.045547:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118660, 118660] 00000080:00200000:2.0:1788201707.045554:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.045556:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.045575:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.045577:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.045587:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486035456, count: 4096 00000080:00200000:2.0:1788201707.045592:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486035456, bytes: 4096 00000080:00200000:2.0:1788201707.045594:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.045596:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.045597:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486035456, 486039552) 0 0 00000080:00200000:2.0:1788201707.045599:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.045602:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118661, 118661] 00020000:00200000:2.0:1788201707.045604:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486035456 chunk: [486035456, 486039552) 486039552, 4096 00020000:00200000:2.0:1788201707.045605:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.045612:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118661, 118661] 00000008:00010000:2.0:1788201707.045615:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.045616:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.045621:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045627:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486035456 486039551) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045636:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045641:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045645:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.045649:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486035456, 486039552) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.045652:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118661 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.045656:0:17463:0:(rw26.c:898:ll_write_end()) pos 486035456, len 4096, copied 4096 00000080:00200000:2.0:1788201707.045659:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000010a736d9[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118661] 00000080:00200000:2.0:1788201707.045661:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000006b34024 fffffc0001029 2:0 ffff93c0c1341e00 118661 lru 00000080:00200000:2.0:1788201707.045664:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000dec07408 118661: 1< 2 - - > 2< 486035456 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70246 - -> 00000080:00200000:2.0:1788201707.045666:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000010a736d9 00000080:00200000:2.0:1788201707.045667:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.045668:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.045673:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.045681:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.045683:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.045684:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.045690:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.045691:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.045693:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.045696:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045700:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.045705:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.045707:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.045709:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.045711:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118661, 118661] 00000080:00200000:2.0:1788201707.045718:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.045719:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.045736:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.045739:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.045748:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486039552, count: 4096 00000080:00200000:2.0:1788201707.045754:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486039552, bytes: 4096 00000080:00200000:2.0:1788201707.045756:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.045758:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.045759:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486039552, 486043648) 0 0 00000080:00200000:2.0:1788201707.045761:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.045764:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118662, 118662] 00020000:00200000:2.0:1788201707.045766:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486039552 chunk: [486039552, 486043648) 486043648, 4096 00020000:00200000:2.0:1788201707.045767:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.045774:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118662, 118662] 00000008:00010000:2.0:1788201707.045777:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.045778:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.045783:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045792:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486039552 486043647) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045797:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045801:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045805:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.045809:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486039552, 486043648) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.045812:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118662 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.045816:0:17463:0:(rw26.c:898:ll_write_end()) pos 486039552, len 4096, copied 4096 00000080:00200000:2.0:1788201707.045818:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009d5be53f[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118662] 00000080:00200000:2.0:1788201707.045819:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006ed5cddc fffffc0001029 2:0 ffff93c0c13411c0 118662 lru 00000080:00200000:2.0:1788201707.045823:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c60e97d5 118662: 1< 2 - - > 2< 486039552 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70247 - -> 00000080:00200000:2.0:1788201707.045825:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009d5be53f 00000080:00200000:2.0:1788201707.045825:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.045827:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.045845:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.045853:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.045854:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.045856:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.045862:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.045863:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.045865:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.045867:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045872:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.045877:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.045878:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.045880:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.045882:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118662, 118662] 00000080:00200000:2.0:1788201707.045889:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.045890:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.045907:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.045909:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.045919:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486043648, count: 4096 00000080:00200000:2.0:1788201707.045925:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486043648, bytes: 4096 00000080:00200000:2.0:1788201707.045926:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.045928:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.045929:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486043648, 486047744) 0 0 00000080:00200000:2.0:1788201707.045931:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.045934:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118663, 118663] 00020000:00200000:2.0:1788201707.045936:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486043648 chunk: [486043648, 486047744) 486047744, 4096 00020000:00200000:2.0:1788201707.045937:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.045944:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118663, 118663] 00000008:00010000:2.0:1788201707.045947:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.045948:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.045954:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045959:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486043648 486047743) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045965:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045969:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.045973:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.045977:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486043648, 486047744) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.045980:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118663 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.045983:0:17463:0:(rw26.c:898:ll_write_end()) pos 486043648, len 4096, copied 4096 00000080:00200000:2.0:1788201707.045985:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008411e469[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118663] 00000080:00200000:2.0:1788201707.045987:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006b6b6086 fffffc0001029 2:0 ffff93c0c1341620 118663 lru 00000080:00200000:2.0:1788201707.045991:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000693fd516 118663: 1< 2 - - > 2< 486043648 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70248 - -> 00000080:00200000:2.0:1788201707.045992:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008411e469 00000080:00200000:2.0:1788201707.045992:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.045993:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.045998:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.046006:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.046022:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.046024:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.046031:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.046033:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.046034:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.046037:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046042:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.046047:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.046048:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.046051:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.046053:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118663, 118663] 00000080:00200000:2.0:1788201707.046060:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.046061:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.046079:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.046081:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.046091:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486047744, count: 4096 00000080:00200000:2.0:1788201707.046097:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486047744, bytes: 4096 00000080:00200000:2.0:1788201707.046098:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.046100:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.046101:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486047744, 486051840) 0 0 00000080:00200000:2.0:1788201707.046103:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.046106:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118664, 118664] 00020000:00200000:2.0:1788201707.046108:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486047744 chunk: [486047744, 486051840) 486051840, 4096 00020000:00200000:2.0:1788201707.046110:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.046116:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118664, 118664] 00000008:00010000:2.0:1788201707.046120:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.046121:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.046126:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046132:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486047744 486051839) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046137:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046141:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046145:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.046149:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486047744, 486051840) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.046160:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118664 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.046166:0:17463:0:(rw26.c:898:ll_write_end()) pos 486047744, len 4096, copied 4096 00000080:00200000:2.0:1788201707.046169:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000084133f5b[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118664] 00000080:00200000:2.0:1788201707.046170:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b3e3af3e fffffc0001029 2:0 ffff93c0c1341d20 118664 lru 00000080:00200000:2.0:1788201707.046174:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009cfad036 118664: 1< 2 - - > 2< 486047744 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70249 - -> 00000080:00200000:2.0:1788201707.046176:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000084133f5b 00000080:00200000:2.0:1788201707.046176:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.046178:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.046183:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.046194:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.046196:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.046197:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.046203:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.046205:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.046206:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.046209:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046213:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.046218:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.046219:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.046222:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.046224:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118664, 118664] 00000080:00200000:2.0:1788201707.046231:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.046233:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.046250:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.046252:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.046262:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486051840, count: 4096 00000080:00200000:2.0:1788201707.046267:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486051840, bytes: 4096 00000080:00200000:2.0:1788201707.046269:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.046271:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.046272:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486051840, 486055936) 0 0 00000080:00200000:2.0:1788201707.046274:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.046277:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118665, 118665] 00020000:00200000:2.0:1788201707.046279:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486051840 chunk: [486051840, 486055936) 486055936, 4096 00020000:00200000:2.0:1788201707.046281:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.046287:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118665, 118665] 00000008:00010000:2.0:1788201707.046290:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.046291:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.046296:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046302:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486051840 486055935) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046307:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046312:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046316:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.046319:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486051840, 486055936) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.046322:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118665 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.046326:0:17463:0:(rw26.c:898:ll_write_end()) pos 486051840, len 4096, copied 4096 00000080:00200000:2.0:1788201707.046328:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000028549cbf[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118665] 00000080:00200000:2.0:1788201707.046330:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000048e38c0e fffffc0001029 2:0 ffff93c0c13419a0 118665 lru 00000080:00200000:2.0:1788201707.046333:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002380eafb 118665: 1< 2 - - > 2< 486051840 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70250 - -> 00000080:00200000:2.0:1788201707.046335:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000028549cbf 00000080:00200000:2.0:1788201707.046336:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.046337:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.046342:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.046350:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.046352:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.046353:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.046358:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.046360:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.046362:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.046364:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046369:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.046373:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.046375:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.046377:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.046379:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118665, 118665] 00000080:00200000:2.0:1788201707.046385:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.046387:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.046405:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.046407:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.046416:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486055936, count: 4096 00000080:00200000:2.0:1788201707.046422:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486055936, bytes: 4096 00000080:00200000:2.0:1788201707.046424:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.046426:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.046427:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486055936, 486060032) 0 0 00000080:00200000:2.0:1788201707.046428:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.046432:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118666, 118666] 00020000:00200000:2.0:1788201707.046434:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486055936 chunk: [486055936, 486060032) 486060032, 4096 00020000:00200000:2.0:1788201707.046435:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.046441:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118666, 118666] 00000008:00010000:2.0:1788201707.046444:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.046446:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.046450:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046456:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486055936 486060031) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046462:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046466:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046470:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.046474:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486055936, 486060032) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.046478:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118666 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.046481:0:17463:0:(rw26.c:898:ll_write_end()) pos 486055936, len 4096, copied 4096 00000080:00200000:2.0:1788201707.046483:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000540b665b[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118666] 00000080:00200000:2.0:1788201707.046485:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b01de4bc fffffc0001029 2:0 ffff93c0c1341b60 118666 lru 00000080:00200000:2.0:1788201707.046490:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000eb0db3dc 118666: 1< 2 - - > 2< 486055936 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70251 - -> 00000080:00200000:2.0:1788201707.046494:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000540b665b 00000080:00200000:2.0:1788201707.046495:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.046497:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.046505:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.046516:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.046519:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.046522:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.046530:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.046533:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.046535:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.046539:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046547:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.046554:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.046555:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.046558:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.046560:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118666, 118666] 00000080:00200000:2.0:1788201707.046567:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.046569:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.046588:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.046590:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.046600:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486060032, count: 4096 00000080:00200000:2.0:1788201707.046605:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486060032, bytes: 4096 00000080:00200000:2.0:1788201707.046607:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.046609:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.046610:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486060032, 486064128) 0 0 00000080:00200000:2.0:1788201707.046612:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.046615:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118667, 118667] 00020000:00200000:2.0:1788201707.046617:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486060032 chunk: [486060032, 486064128) 486064128, 4096 00020000:00200000:2.0:1788201707.046618:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.046625:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118667, 118667] 00000008:00010000:2.0:1788201707.046628:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.046629:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.046634:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046640:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486060032 486064127) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046646:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046651:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046654:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.046658:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486060032, 486064128) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.046661:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118667 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.046665:0:17463:0:(rw26.c:898:ll_write_end()) pos 486060032, len 4096, copied 4096 00000080:00200000:2.0:1788201707.046667:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000056654c5a[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118667] 00000080:00200000:2.0:1788201707.046669:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002932c63b fffffc0001029 2:0 ffff93c0c1341c40 118667 lru 00000080:00200000:2.0:1788201707.046673:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000abdd028f 118667: 1< 2 - - > 2< 486060032 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70252 - -> 00000080:00200000:2.0:1788201707.046674:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000056654c5a 00000080:00200000:2.0:1788201707.046675:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.046676:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.046681:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.046688:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.046690:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.046691:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.046697:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.046699:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.046700:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.046703:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046707:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.046712:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.046713:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.046716:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.046718:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118667, 118667] 00000080:00200000:2.0:1788201707.046724:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.046726:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.046743:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.046745:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.046755:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486064128, count: 4096 00000080:00200000:2.0:1788201707.046761:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486064128, bytes: 4096 00000080:00200000:2.0:1788201707.046762:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.046764:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.046765:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486064128, 486068224) 0 0 00000080:00200000:2.0:1788201707.046767:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.046770:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118668, 118668] 00020000:00200000:2.0:1788201707.046772:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486064128 chunk: [486064128, 486068224) 486068224, 4096 00020000:00200000:2.0:1788201707.046773:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.046780:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118668, 118668] 00000008:00010000:2.0:1788201707.046783:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.046784:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.046789:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046795:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486064128 486068223) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046800:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046804:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046808:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.046812:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486064128, 486068224) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.046815:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118668 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.046819:0:17463:0:(rw26.c:898:ll_write_end()) pos 486064128, len 4096, copied 4096 00000080:00200000:2.0:1788201707.046821:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f5bcdb7d[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118668] 00000080:00200000:2.0:1788201707.046823:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005b743fed fffffc0001029 2:0 ffff93c0c1341540 118668 lru 00000080:00200000:2.0:1788201707.046826:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000043681abe 118668: 1< 2 - - > 2< 486064128 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70253 - -> 00000080:00200000:2.0:1788201707.046828:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f5bcdb7d 00000080:00200000:2.0:1788201707.046841:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.046843:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.046848:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.046856:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.046858:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.046860:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.046865:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.046867:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.046868:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.046871:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046875:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.046880:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.046881:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.046884:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.046886:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118668, 118668] 00000080:00200000:2.0:1788201707.046892:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.046894:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.046911:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.046914:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.046923:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486068224, count: 4096 00000080:00200000:2.0:1788201707.046929:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486068224, bytes: 4096 00000080:00200000:2.0:1788201707.046931:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.046933:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.046934:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486068224, 486072320) 0 0 00000080:00200000:2.0:1788201707.046935:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.046939:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118669, 118669] 00020000:00200000:2.0:1788201707.046941:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486068224 chunk: [486068224, 486072320) 486072320, 4096 00020000:00200000:2.0:1788201707.046942:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.046948:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118669, 118669] 00000008:00010000:2.0:1788201707.046951:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.046953:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.046958:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046963:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486068224 486072319) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046969:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046973:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.046977:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.046981:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486068224, 486072320) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.046984:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118669 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.046988:0:17463:0:(rw26.c:898:ll_write_end()) pos 486068224, len 4096, copied 4096 00000080:00200000:2.0:1788201707.046990:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002373599d[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118669] 00000080:00200000:2.0:1788201707.046992:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000033aadb59 fffffc0001029 2:0 ffff93c0c1341460 118669 lru 00000080:00200000:2.0:1788201707.046996:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000aec84f57 118669: 1< 2 - - > 2< 486068224 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70254 - -> 00000080:00200000:2.0:1788201707.046997:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002373599d 00000080:00200000:2.0:1788201707.046998:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.046999:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.047004:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.047026:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.047028:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.047029:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.047036:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.047038:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.047039:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.047042:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047047:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.047052:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.047053:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.047055:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.047057:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118669, 118669] 00000080:00200000:2.0:1788201707.047064:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.047066:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.047084:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.047086:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.047095:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486072320, count: 4096 00000080:00200000:2.0:1788201707.047101:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486072320, bytes: 4096 00000080:00200000:2.0:1788201707.047103:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.047105:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.047106:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486072320, 486076416) 0 0 00000080:00200000:2.0:1788201707.047107:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.047111:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118670, 118670] 00020000:00200000:2.0:1788201707.047112:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486072320 chunk: [486072320, 486076416) 486076416, 4096 00020000:00200000:2.0:1788201707.047114:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.047120:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118670, 118670] 00000008:00010000:2.0:1788201707.047123:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.047125:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.047130:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047136:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486072320 486076415) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047141:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047145:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047149:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.047156:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486072320, 486076416) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.047159:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118670 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.047163:0:17463:0:(rw26.c:898:ll_write_end()) pos 486072320, len 4096, copied 4096 00000080:00200000:2.0:1788201707.047165:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f562362d[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118670] 00000080:00200000:2.0:1788201707.047167:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000039b5b4f2 fffffc0001029 2:0 ffff93c0c13410e0 118670 lru 00000080:00200000:2.0:1788201707.047170:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ce89b61f 118670: 1< 2 - - > 2< 486072320 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70255 - -> 00000080:00200000:2.0:1788201707.047172:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f562362d 00000080:00200000:2.0:1788201707.047172:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.047174:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.047179:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.047186:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.047188:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.047189:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.047194:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.047196:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.047198:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.047200:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047204:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.047209:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.047210:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.047213:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.047215:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118670, 118670] 00000080:00200000:2.0:1788201707.047221:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.047223:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.047240:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.047242:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.047252:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486076416, count: 4096 00000080:00200000:2.0:1788201707.047258:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486076416, bytes: 4096 00000080:00200000:2.0:1788201707.047259:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.047262:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.047263:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486076416, 486080512) 0 0 00000080:00200000:2.0:1788201707.047264:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.047267:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118671, 118671] 00020000:00200000:2.0:1788201707.047269:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486076416 chunk: [486076416, 486080512) 486080512, 4096 00020000:00200000:2.0:1788201707.047271:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.047277:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118671, 118671] 00000008:00010000:2.0:1788201707.047280:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.047281:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.047286:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047292:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486076416 486080511) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047297:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047301:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047309:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.047313:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486076416, 486080512) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.047316:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118671 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.047319:0:17463:0:(rw26.c:898:ll_write_end()) pos 486076416, len 4096, copied 4096 00000080:00200000:2.0:1788201707.047321:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000181ba7c5[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118671] 00000080:00200000:2.0:1788201707.047323:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a0b61e1f fffffc0001029 2:0 ffff93c0c1341700 118671 lru 00000080:00200000:2.0:1788201707.047326:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000056a57de6 118671: 1< 2 - - > 2< 486076416 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70256 - -> 00000080:00200000:2.0:1788201707.047328:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000181ba7c5 00000080:00200000:2.0:1788201707.047328:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.047329:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.047335:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.047342:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.047344:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.047345:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.047351:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.047352:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.047354:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.047356:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047361:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.047369:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.047371:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.047373:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.047375:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118671, 118671] 00000080:00200000:2.0:1788201707.047381:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.047383:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.047400:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.047403:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.047412:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486080512, count: 4096 00000080:00200000:2.0:1788201707.047418:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486080512, bytes: 4096 00000080:00200000:2.0:1788201707.047420:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.047423:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.047424:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486080512, 486084608) 0 0 00000080:00200000:2.0:1788201707.047426:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.047431:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118672, 118672] 00020000:00200000:2.0:1788201707.047434:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486080512 chunk: [486080512, 486084608) 486084608, 4096 00020000:00200000:2.0:1788201707.047436:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.047447:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118672, 118672] 00000008:00010000:2.0:1788201707.047452:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.047454:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.047461:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047471:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486080512 486084607) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047479:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047488:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047492:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.047496:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486080512, 486084608) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.047500:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118672 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.047534:0:17463:0:(rw26.c:898:ll_write_end()) pos 486080512, len 4096, copied 4096 00000080:00200000:2.0:1788201707.047537:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e65f196a[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118672] 00000080:00200000:2.0:1788201707.047539:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f86f470a fffffc0001029 2:0 ffff93c0c1342a80 118672 lru 00000080:00200000:2.0:1788201707.047542:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b9b80672 118672: 1< 2 - - > 2< 486080512 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70257 - -> 00000080:00200000:2.0:1788201707.047543:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e65f196a 00000080:00200000:2.0:1788201707.047544:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.047545:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.047552:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.047560:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.047561:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.047563:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.047569:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.047571:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.047572:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.047575:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047583:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.047588:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.047589:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.047591:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.047593:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118672, 118672] 00000080:00200000:2.0:1788201707.047600:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.047602:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.047620:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.047622:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.047632:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486084608, count: 4096 00000080:00200000:2.0:1788201707.047637:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486084608, bytes: 4096 00000080:00200000:2.0:1788201707.047639:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.047641:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.047643:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486084608, 486088704) 0 0 00000080:00200000:2.0:1788201707.047644:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.047647:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118673, 118673] 00020000:00200000:2.0:1788201707.047649:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486084608 chunk: [486084608, 486088704) 486088704, 4096 00020000:00200000:2.0:1788201707.047651:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.047657:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118673, 118673] 00000008:00010000:2.0:1788201707.047660:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.047662:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.047667:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047672:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486084608 486088703) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047681:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047686:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047689:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.047694:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486084608, 486088704) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.047697:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118673 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.047701:0:17463:0:(rw26.c:898:ll_write_end()) pos 486084608, len 4096, copied 4096 00000080:00200000:2.0:1788201707.047703:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000dea88fdf[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118673] 00000080:00200000:2.0:1788201707.047705:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b445b23a fffffc0001029 2:0 ffff93c0c1342380 118673 lru 00000080:00200000:2.0:1788201707.047708:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a22f439b 118673: 1< 2 - - > 2< 486084608 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70258 - -> 00000080:00200000:2.0:1788201707.047710:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000dea88fdf 00000080:00200000:2.0:1788201707.047711:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.047712:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.047720:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.047731:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.047734:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.047736:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.047745:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.047747:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.047750:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.047755:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047762:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.047770:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.047772:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.047776:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.047779:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118673, 118673] 00000080:00200000:2.0:1788201707.047788:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.047791:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.047817:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.047821:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.047852:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486088704, count: 4096 00000080:00200000:2.0:1788201707.047864:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486088704, bytes: 4096 00000080:00200000:2.0:1788201707.047866:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.047869:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.047871:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486088704, 486092800) 0 0 00000080:00200000:2.0:1788201707.047873:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.047877:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118674, 118674] 00020000:00200000:2.0:1788201707.047880:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486088704 chunk: [486088704, 486092800) 486092800, 4096 00020000:00200000:2.0:1788201707.047882:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.047893:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118674, 118674] 00000008:00010000:2.0:1788201707.047899:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.047901:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.047908:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047923:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486088704 486092799) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047933:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047940:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.047947:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.047953:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486088704, 486092800) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.047957:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118674 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.047961:0:17463:0:(rw26.c:898:ll_write_end()) pos 486088704, len 4096, copied 4096 00000080:00200000:2.0:1788201707.047964:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000f3df8c1[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118674] 00000080:00200000:2.0:1788201707.047966:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008bb407d4 fffffc0001029 2:0 ffff93c0c13428c0 118674 lru 00000080:00200000:2.0:1788201707.047969:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c75d9829 118674: 1< 2 - - > 2< 486088704 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70259 - -> 00000080:00200000:2.0:1788201707.047971:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000f3df8c1 00000080:00200000:2.0:1788201707.047971:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.047973:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.047979:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.047986:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.047988:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.047990:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.047996:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.047998:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.048000:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.048002:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048021:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.048029:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.048030:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.048034:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.048036:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118674, 118674] 00000080:00200000:2.0:1788201707.048043:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.048045:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.048064:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.048066:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.048076:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486092800, count: 4096 00000080:00200000:2.0:1788201707.048082:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486092800, bytes: 4096 00000080:00200000:2.0:1788201707.048084:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.048087:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.048088:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486092800, 486096896) 0 0 00000080:00200000:2.0:1788201707.048089:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.048093:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118675, 118675] 00020000:00200000:2.0:1788201707.048096:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486092800 chunk: [486092800, 486096896) 486096896, 4096 00020000:00200000:2.0:1788201707.048098:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.048109:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118675, 118675] 00000008:00010000:2.0:1788201707.048115:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.048116:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.048124:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048133:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486092800 486096895) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048142:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048150:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048158:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.048166:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486092800, 486096896) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.048171:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118675 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.048176:0:17463:0:(rw26.c:898:ll_write_end()) pos 486092800, len 4096, copied 4096 00000080:00200000:2.0:1788201707.048180:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c9c8edf4[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118675] 00000080:00200000:2.0:1788201707.048183:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c9c4e616 fffffc0001029 2:0 ffff93c0c1342ee0 118675 lru 00000080:00200000:2.0:1788201707.048188:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003932a3c0 118675: 1< 2 - - > 2< 486092800 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70260 - -> 00000080:00200000:2.0:1788201707.048190:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c9c8edf4 00000080:00200000:2.0:1788201707.048191:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.048192:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.048200:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.048209:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.048211:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.048213:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.048220:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.048222:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.048223:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.048226:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048231:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.048236:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.048237:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.048240:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.048242:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118675, 118675] 00000080:00200000:2.0:1788201707.048251:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.048254:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.048281:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.048285:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.048301:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486096896, count: 4096 00000080:00200000:2.0:1788201707.048312:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486096896, bytes: 4096 00000080:00200000:2.0:1788201707.048315:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.048319:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.048321:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486096896, 486100992) 0 0 00000080:00200000:2.0:1788201707.048323:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.048328:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118676, 118676] 00020000:00200000:2.0:1788201707.048331:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486096896 chunk: [486096896, 486100992) 486100992, 4096 00020000:00200000:2.0:1788201707.048333:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.048343:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118676, 118676] 00000008:00010000:2.0:1788201707.048348:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.048350:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.048357:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048366:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486096896 486100991) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048373:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048378:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048382:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.048387:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486096896, 486100992) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.048390:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118676 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.048394:0:17463:0:(rw26.c:898:ll_write_end()) pos 486096896, len 4096, copied 4096 00000080:00200000:2.0:1788201707.048396:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d7a22e98[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118676] 00000080:00200000:2.0:1788201707.048398:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000aacaf899 fffffc0001029 2:0 ffff93c0c13427e0 118676 lru 00000080:00200000:2.0:1788201707.048401:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f9c8659d 118676: 1< 2 - - > 2< 486096896 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70261 - -> 00000080:00200000:2.0:1788201707.048403:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d7a22e98 00000080:00200000:2.0:1788201707.048403:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.048405:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.048410:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.048422:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.048425:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.048427:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.048435:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.048438:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.048441:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.048444:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048453:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.048461:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.048463:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.048468:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.048471:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118676, 118676] 00000080:00200000:2.0:1788201707.048479:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.048481:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.048500:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.048503:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.048513:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486100992, count: 4096 00000080:00200000:2.0:1788201707.048519:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486100992, bytes: 4096 00000080:00200000:2.0:1788201707.048520:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.048523:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.048524:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486100992, 486105088) 0 0 00000080:00200000:2.0:1788201707.048525:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.048529:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118677, 118677] 00020000:00200000:2.0:1788201707.048531:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486100992 chunk: [486100992, 486105088) 486105088, 4096 00020000:00200000:2.0:1788201707.048532:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.048539:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118677, 118677] 00000008:00010000:2.0:1788201707.048542:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.048544:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.048549:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048555:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486100992 486105087) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048560:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048565:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048568:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.048573:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486100992, 486105088) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.048576:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118677 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.048580:0:17463:0:(rw26.c:898:ll_write_end()) pos 486100992, len 4096, copied 4096 00000080:00200000:2.0:1788201707.048583:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000023bddf5d[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118677] 00000080:00200000:2.0:1788201707.048585:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000751ed771 fffffc0001029 2:0 ffff93c0c13422a0 118677 lru 00000080:00200000:2.0:1788201707.048588:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d81f1768 118677: 1< 2 - - > 2< 486100992 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70262 - -> 00000080:00200000:2.0:1788201707.048590:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000023bddf5d 00000080:00200000:2.0:1788201707.048590:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.048592:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.048597:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.048605:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.048607:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.048608:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.048614:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.048616:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.048617:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.048620:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048624:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.048629:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.048630:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.048633:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.048635:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118677, 118677] 00000080:00200000:2.0:1788201707.048641:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.048643:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.048660:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.048662:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.048672:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486105088, count: 4096 00000080:00200000:2.0:1788201707.048677:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486105088, bytes: 4096 00000080:00200000:2.0:1788201707.048679:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.048681:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.048682:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486105088, 486109184) 0 0 00000080:00200000:2.0:1788201707.048684:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.048687:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118678, 118678] 00020000:00200000:2.0:1788201707.048689:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486105088 chunk: [486105088, 486109184) 486109184, 4096 00020000:00200000:2.0:1788201707.048691:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.048697:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118678, 118678] 00000008:00010000:2.0:1788201707.048700:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.048701:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.048706:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048712:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486105088 486109183) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048717:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048722:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048726:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.048730:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486105088, 486109184) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.048733:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118678 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.048737:0:17463:0:(rw26.c:898:ll_write_end()) pos 486105088, len 4096, copied 4096 00000080:00200000:2.0:1788201707.048739:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f4e12622[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118678] 00000080:00200000:2.0:1788201707.048741:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c2ed162c fffffc0001029 2:0 ffff93c0c1342e00 118678 lru 00000080:00200000:2.0:1788201707.048745:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d37ab167 118678: 1< 2 - - > 2< 486105088 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70263 - -> 00000080:00200000:2.0:1788201707.048747:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f4e12622 00000080:00200000:2.0:1788201707.048748:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.048749:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.048754:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.048762:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.048763:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.048765:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.048770:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.048772:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.048773:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.048776:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048780:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.048784:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.048786:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.048788:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.048790:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118678, 118678] 00000080:00200000:2.0:1788201707.048797:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.048799:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.048816:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.048818:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.048828:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486109184, count: 4096 00000080:00200000:2.0:1788201707.048847:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486109184, bytes: 4096 00000080:00200000:2.0:1788201707.048849:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.048851:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.048852:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486109184, 486113280) 0 0 00000080:00200000:2.0:1788201707.048854:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.048857:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118679, 118679] 00020000:00200000:2.0:1788201707.048859:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486109184 chunk: [486109184, 486113280) 486113280, 4096 00020000:00200000:2.0:1788201707.048860:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.048868:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118679, 118679] 00000008:00010000:2.0:1788201707.048872:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.048874:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.048881:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048890:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486109184 486113279) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048898:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048906:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048913:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.048921:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486109184, 486113280) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.048926:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118679 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.048931:0:17463:0:(rw26.c:898:ll_write_end()) pos 486109184, len 4096, copied 4096 00000080:00200000:2.0:1788201707.048936:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000033d5647b[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118679] 00000080:00200000:2.0:1788201707.048939:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b5d56398 fffffc0001029 2:0 ffff93c0c13421c0 118679 lru 00000080:00200000:2.0:1788201707.048944:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009d2f063a 118679: 1< 2 - - > 2< 486109184 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70264 - -> 00000080:00200000:2.0:1788201707.048946:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000033d5647b 00000080:00200000:2.0:1788201707.048947:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.048949:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.048958:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.048966:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.048968:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.048970:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.048976:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.048978:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.048980:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.048982:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.048987:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.048993:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.048994:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.048996:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.048998:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118679, 118679] 00000080:00200000:2.0:1788201707.049005:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.049007:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.049039:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.049041:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.049052:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486113280, count: 4096 00000080:00200000:2.0:1788201707.049058:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486113280, bytes: 4096 00000080:00200000:2.0:1788201707.049060:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.049062:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.049063:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486113280, 486117376) 0 0 00000080:00200000:2.0:1788201707.049064:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.049068:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118680, 118680] 00020000:00200000:2.0:1788201707.049070:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486113280 chunk: [486113280, 486117376) 486117376, 4096 00020000:00200000:2.0:1788201707.049071:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.049078:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118680, 118680] 00000008:00010000:2.0:1788201707.049081:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.049083:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.049088:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049094:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486113280 486117375) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049099:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049104:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049108:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.049113:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486113280, 486117376) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.049116:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118680 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.049120:0:17463:0:(rw26.c:898:ll_write_end()) pos 486113280, len 4096, copied 4096 00000080:00200000:2.0:1788201707.049122:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d7cb2021[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118680] 00000080:00200000:2.0:1788201707.049124:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c893e135 fffffc0001029 2:0 ffff93c0c1342620 118680 lru 00000080:00200000:2.0:1788201707.049128:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e9571db6 118680: 1< 2 - - > 2< 486113280 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70265 - -> 00000080:00200000:2.0:1788201707.049129:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d7cb2021 00000080:00200000:2.0:1788201707.049130:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.049131:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.049137:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.049145:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.049146:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.049148:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.049154:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.049156:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.049157:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.049160:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049164:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.049169:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.049170:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.049173:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.049175:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118680, 118680] 00000080:00200000:2.0:1788201707.049181:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.049183:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.049200:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.049203:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.049212:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486117376, count: 4096 00000080:00200000:2.0:1788201707.049218:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486117376, bytes: 4096 00000080:00200000:2.0:1788201707.049220:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.049222:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.049223:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486117376, 486121472) 0 0 00000080:00200000:2.0:1788201707.049225:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.049228:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118681, 118681] 00020000:00200000:2.0:1788201707.049230:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486117376 chunk: [486117376, 486121472) 486121472, 4096 00020000:00200000:2.0:1788201707.049231:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.049237:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118681, 118681] 00000008:00010000:2.0:1788201707.049240:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.049242:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.049246:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049252:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486117376 486121471) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049258:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049262:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049266:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.049270:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486117376, 486121472) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.049273:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118681 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.049277:0:17463:0:(rw26.c:898:ll_write_end()) pos 486117376, len 4096, copied 4096 00000080:00200000:2.0:1788201707.049279:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c2dafcd5[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118681] 00000080:00200000:2.0:1788201707.049281:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000066279883 fffffc0001029 2:0 ffff93c0c1342d20 118681 lru 00000080:00200000:2.0:1788201707.049284:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e18c9d98 118681: 1< 2 - - > 2< 486117376 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70266 - -> 00000080:00200000:2.0:1788201707.049286:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c2dafcd5 00000080:00200000:2.0:1788201707.049286:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.049288:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.049293:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.049300:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.049302:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.049303:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.049309:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.049310:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.049312:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.049314:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049318:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.049323:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.049324:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.049327:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.049329:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118681, 118681] 00000080:00200000:2.0:1788201707.049335:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.049337:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.049354:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.049356:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.049366:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486121472, count: 4096 00000080:00200000:2.0:1788201707.049372:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486121472, bytes: 4096 00000080:00200000:2.0:1788201707.049374:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.049376:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.049377:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486121472, 486125568) 0 0 00000080:00200000:2.0:1788201707.049378:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.049381:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118682, 118682] 00020000:00200000:2.0:1788201707.049383:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486121472 chunk: [486121472, 486125568) 486125568, 4096 00020000:00200000:2.0:1788201707.049385:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.049391:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118682, 118682] 00000008:00010000:2.0:1788201707.049394:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.049395:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.049400:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049406:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486121472 486125567) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049411:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049415:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049419:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.049432:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486121472, 486125568) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.049437:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118682 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.049441:0:17463:0:(rw26.c:898:ll_write_end()) pos 486121472, len 4096, copied 4096 00000080:00200000:2.0:1788201707.049445:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000a3421e4[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118682] 00000080:00200000:2.0:1788201707.049448:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006eeb6ec5 fffffc0001029 2:0 ffff93c0c13429a0 118682 lru 00000080:00200000:2.0:1788201707.049454:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000116f16a7 118682: 1< 2 - - > 2< 486121472 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70267 - -> 00000080:00200000:2.0:1788201707.049456:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000a3421e4 00000080:00200000:2.0:1788201707.049457:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.049459:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.049468:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.049479:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.049481:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.049482:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.049488:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.049490:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.049492:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.049494:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049499:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.049504:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.049506:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.049509:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.049511:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118682, 118682] 00000080:00200000:2.0:1788201707.049517:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.049519:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.049537:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.049539:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.049549:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486125568, count: 4096 00000080:00200000:2.0:1788201707.049555:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486125568, bytes: 4096 00000080:00200000:2.0:1788201707.049557:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.049559:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.049560:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486125568, 486129664) 0 0 00000080:00200000:2.0:1788201707.049561:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.049565:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118683, 118683] 00020000:00200000:2.0:1788201707.049567:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486125568 chunk: [486125568, 486129664) 486129664, 4096 00020000:00200000:2.0:1788201707.049568:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.049575:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118683, 118683] 00000008:00010000:2.0:1788201707.049578:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.049579:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.049584:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049590:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486125568 486129663) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049595:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049599:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049607:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.049611:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486125568, 486129664) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.049614:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118683 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.049617:0:17463:0:(rw26.c:898:ll_write_end()) pos 486125568, len 4096, copied 4096 00000080:00200000:2.0:1788201707.049620:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000959361ab[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118683] 00000080:00200000:2.0:1788201707.049621:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003d4282d8 fffffc0001029 2:0 ffff93c0c1342b60 118683 lru 00000080:00200000:2.0:1788201707.049625:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f8d49190 118683: 1< 2 - - > 2< 486125568 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70268 - -> 00000080:00200000:2.0:1788201707.049626:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000959361ab 00000080:00200000:2.0:1788201707.049627:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.049628:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.049633:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.049641:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.049643:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.049644:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.049650:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.049651:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.049653:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.049655:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049659:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.049668:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.049669:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.049672:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.049674:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118683, 118683] 00000080:00200000:2.0:1788201707.049680:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.049682:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.049699:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.049701:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.049711:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486129664, count: 4096 00000080:00200000:2.0:1788201707.049716:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486129664, bytes: 4096 00000080:00200000:2.0:1788201707.049718:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.049720:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.049721:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486129664, 486133760) 0 0 00000080:00200000:2.0:1788201707.049723:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.049726:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118684, 118684] 00020000:00200000:2.0:1788201707.049728:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486129664 chunk: [486129664, 486133760) 486133760, 4096 00020000:00200000:2.0:1788201707.049729:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.049735:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118684, 118684] 00000008:00010000:2.0:1788201707.049738:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.049740:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.049744:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049750:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486129664 486133759) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049755:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049763:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049768:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.049772:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486129664, 486133760) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.049774:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118684 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.049778:0:17463:0:(rw26.c:898:ll_write_end()) pos 486129664, len 4096, copied 4096 00000080:00200000:2.0:1788201707.049780:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a7fa7aa5[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118684] 00000080:00200000:2.0:1788201707.049782:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e03baa11 fffffc0001029 2:0 ffff93c0c1342c40 118684 lru 00000080:00200000:2.0:1788201707.049785:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007c0fa39e 118684: 1< 2 - - > 2< 486129664 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70269 - -> 00000080:00200000:2.0:1788201707.049786:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a7fa7aa5 00000080:00200000:2.0:1788201707.049787:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.049788:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.049793:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.049801:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.049802:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.049803:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.049809:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.049810:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.049812:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.049814:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049823:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.049827:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.049847:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.049850:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.049852:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118684, 118684] 00000080:00200000:2.0:1788201707.049858:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.049860:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.049881:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.049883:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.049894:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486133760, count: 4096 00000080:00200000:2.0:1788201707.049900:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486133760, bytes: 4096 00000080:00200000:2.0:1788201707.049902:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.049904:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.049905:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486133760, 486137856) 0 0 00000080:00200000:2.0:1788201707.049907:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.049910:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118685, 118685] 00020000:00200000:2.0:1788201707.049912:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486133760 chunk: [486133760, 486137856) 486137856, 4096 00020000:00200000:2.0:1788201707.049913:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.049919:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118685, 118685] 00000008:00010000:2.0:1788201707.049922:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.049924:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.049929:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049935:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486133760 486137855) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049944:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049948:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.049952:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.049956:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486133760, 486137856) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.049959:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118685 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.049962:0:17463:0:(rw26.c:898:ll_write_end()) pos 486133760, len 4096, copied 4096 00000080:00200000:2.0:1788201707.049965:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ce1970b2[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118685] 00000080:00200000:2.0:1788201707.049966:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bed4d66a fffffc0001029 2:0 ffff93c0c1342540 118685 lru 00000080:00200000:2.0:1788201707.049970:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000036de71f0 118685: 1< 2 - - > 2< 486133760 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70270 - -> 00000080:00200000:2.0:1788201707.049971:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ce1970b2 00000080:00200000:2.0:1788201707.049972:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.049973:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.049978:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.049986:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.049987:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.049989:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.049995:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.049996:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.049998:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.050000:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050005:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.050023:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.050025:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.050028:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.050030:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118685, 118685] 00000080:00200000:2.0:1788201707.050037:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.050039:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.050057:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.050059:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.050069:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486137856, count: 4096 00000080:00200000:2.0:1788201707.050075:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486137856, bytes: 4096 00000080:00200000:2.0:1788201707.050077:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.050079:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.050080:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486137856, 486141952) 0 0 00000080:00200000:2.0:1788201707.050081:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.050084:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118686, 118686] 00020000:00200000:2.0:1788201707.050086:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486137856 chunk: [486137856, 486141952) 486141952, 4096 00020000:00200000:2.0:1788201707.050088:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.050094:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118686, 118686] 00000008:00010000:2.0:1788201707.050097:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.050099:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.050104:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050113:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486137856 486141951) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050118:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050122:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050126:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.050131:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486137856, 486141952) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.050134:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118686 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.050138:0:17463:0:(rw26.c:898:ll_write_end()) pos 486137856, len 4096, copied 4096 00000080:00200000:2.0:1788201707.050140:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000094217f57[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118686] 00000080:00200000:2.0:1788201707.050141:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000460f6c7 fffffc0001029 2:0 ffff93c0c1342460 118686 lru 00000080:00200000:2.0:1788201707.050145:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f4475a00 118686: 1< 2 - - > 2< 486137856 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70271 - -> 00000080:00200000:2.0:1788201707.050147:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000094217f57 00000080:00200000:2.0:1788201707.050147:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.050149:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.050154:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.050161:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.050163:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.050165:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.050170:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.050172:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.050173:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.050176:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050180:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.050185:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.050186:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.050189:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.050191:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118686, 118686] 00000080:00200000:2.0:1788201707.050197:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.050199:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.050217:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.050219:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.050229:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486141952, count: 4096 00000080:00200000:2.0:1788201707.050234:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486141952, bytes: 4096 00000080:00200000:2.0:1788201707.050236:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.050238:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.050239:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486141952, 486146048) 0 0 00000080:00200000:2.0:1788201707.050241:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.050244:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118687, 118687] 00020000:00200000:2.0:1788201707.050246:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486141952 chunk: [486141952, 486146048) 486146048, 4096 00020000:00200000:2.0:1788201707.050247:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.050254:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118687, 118687] 00000008:00010000:2.0:1788201707.050257:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.050258:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.050263:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050269:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486141952 486146047) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050274:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050278:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050282:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.050287:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486141952, 486146048) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.050290:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118687 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.050293:0:17463:0:(rw26.c:898:ll_write_end()) pos 486141952, len 4096, copied 4096 00000080:00200000:2.0:1788201707.050296:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009a578b45[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118687] 00000080:00200000:2.0:1788201707.050297:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000083c52644 fffffc0001029 2:0 ffff93c0c13420e0 118687 lru 00000080:00200000:2.0:1788201707.050301:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005ffafe47 118687: 1< 2 - - > 2< 486141952 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70272 - -> 00000080:00200000:2.0:1788201707.050302:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009a578b45 00000080:00200000:2.0:1788201707.050303:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.050304:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.050309:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.050317:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.050319:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.050320:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.050326:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.050327:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.050329:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.050331:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050336:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.050341:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.050342:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.050344:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.050346:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118687, 118687] 00000080:00200000:2.0:1788201707.050352:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.050354:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.050371:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.050373:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.050383:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486146048, count: 4096 00000080:00200000:2.0:1788201707.050389:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486146048, bytes: 4096 00000080:00200000:2.0:1788201707.050390:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.050392:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.050393:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486146048, 486150144) 0 0 00000080:00200000:2.0:1788201707.050395:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.050398:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118688, 118688] 00020000:00200000:2.0:1788201707.050400:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486146048 chunk: [486146048, 486150144) 486150144, 4096 00020000:00200000:2.0:1788201707.050401:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.050407:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118688, 118688] 00000008:00010000:2.0:1788201707.050411:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.050412:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.050417:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050425:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486146048 486150143) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050434:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050441:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050448:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.050456:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486146048, 486150144) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.050461:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118688 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.050466:0:17463:0:(rw26.c:898:ll_write_end()) pos 486146048, len 4096, copied 4096 00000080:00200000:2.0:1788201707.050470:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007a403b48[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118688] 00000080:00200000:2.0:1788201707.050473:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e9121cca fffffc0001029 2:0 ffff93c0c1342700 118688 lru 00000080:00200000:2.0:1788201707.050476:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000042d116e3 118688: 1< 2 - - > 2< 486146048 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70273 - -> 00000080:00200000:2.0:1788201707.050478:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007a403b48 00000080:00200000:2.0:1788201707.050479:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.050480:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.050486:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.050497:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.050499:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.050501:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.050507:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.050508:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.050510:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.050512:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050517:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.050522:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.050524:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.050526:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.050528:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118688, 118688] 00000080:00200000:2.0:1788201707.050535:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.050537:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.050554:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.050556:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.050566:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486150144, count: 4096 00000080:00200000:2.0:1788201707.050572:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486150144, bytes: 4096 00000080:00200000:2.0:1788201707.050574:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.050576:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.050577:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486150144, 486154240) 0 0 00000080:00200000:2.0:1788201707.050579:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.050582:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118689, 118689] 00020000:00200000:2.0:1788201707.050584:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486150144 chunk: [486150144, 486154240) 486154240, 4096 00020000:00200000:2.0:1788201707.050585:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.050592:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118689, 118689] 00000008:00010000:2.0:1788201707.050595:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.050597:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.050602:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050607:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486150144 486154239) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050612:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050617:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050621:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.050625:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486150144, 486154240) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.050627:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118689 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.050631:0:17463:0:(rw26.c:898:ll_write_end()) pos 486150144, len 4096, copied 4096 00000080:00200000:2.0:1788201707.050633:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000000cd36ed[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118689] 00000080:00200000:2.0:1788201707.050635:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003c4989a1 fffffc0001029 2:0 ffff93c0c1342000 118689 lru 00000080:00200000:2.0:1788201707.050638:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001bfd28c5 118689: 1< 2 - - > 2< 486150144 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70274 - -> 00000080:00200000:2.0:1788201707.050640:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000000cd36ed 00000080:00200000:2.0:1788201707.050640:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.050642:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.050647:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.050655:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.050656:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.050658:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.050663:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.050665:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.050667:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.050669:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050674:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.050678:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.050680:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.050682:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.050684:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118689, 118689] 00000080:00200000:2.0:1788201707.050690:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.050692:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.050709:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.050711:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.050720:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486154240, count: 4096 00000080:00200000:2.0:1788201707.050726:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486154240, bytes: 4096 00000080:00200000:2.0:1788201707.050728:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.050730:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.050731:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486154240, 486158336) 0 0 00000080:00200000:2.0:1788201707.050733:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.050736:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118690, 118690] 00020000:00200000:2.0:1788201707.050738:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486154240 chunk: [486154240, 486158336) 486158336, 4096 00020000:00200000:2.0:1788201707.050740:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.050746:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118690, 118690] 00000008:00010000:2.0:1788201707.050749:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.050750:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.050755:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050761:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486154240 486158335) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050766:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050771:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050775:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.050779:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486154240, 486158336) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.050782:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118690 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.050786:0:17463:0:(rw26.c:898:ll_write_end()) pos 486154240, len 4096, copied 4096 00000080:00200000:2.0:1788201707.050788:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000adb7e35d[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118690] 00000080:00200000:2.0:1788201707.050790:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004f65595c fffffc0001029 2:0 ffff93c0c1343d20 118690 lru 00000080:00200000:2.0:1788201707.050793:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ef80f825 118690: 1< 2 - - > 2< 486154240 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70275 - -> 00000080:00200000:2.0:1788201707.050796:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000adb7e35d 00000080:00200000:2.0:1788201707.050797:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.050798:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.050803:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.050810:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.050812:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.050814:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.050819:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.050821:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.050822:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.050825:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050844:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.050850:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.050851:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.050855:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.050857:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118690, 118690] 00000080:00200000:2.0:1788201707.050863:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.050865:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.050883:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.050885:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.050895:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486158336, count: 4096 00000080:00200000:2.0:1788201707.050900:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486158336, bytes: 4096 00000080:00200000:2.0:1788201707.050902:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.050905:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.050906:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486158336, 486162432) 0 0 00000080:00200000:2.0:1788201707.050907:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.050910:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118691, 118691] 00020000:00200000:2.0:1788201707.050912:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486158336 chunk: [486158336, 486162432) 486162432, 4096 00020000:00200000:2.0:1788201707.050913:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.050920:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118691, 118691] 00000008:00010000:2.0:1788201707.050923:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.050924:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.050929:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050935:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486158336 486162431) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050940:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050945:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.050949:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.050953:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486158336, 486162432) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.050956:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118691 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.050960:0:17463:0:(rw26.c:898:ll_write_end()) pos 486158336, len 4096, copied 4096 00000080:00200000:2.0:1788201707.050962:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a166d150[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118691] 00000080:00200000:2.0:1788201707.050964:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000847a6fb5 fffffc0001029 2:0 ffff93c0c11af9a0 118691 lru 00000080:00200000:2.0:1788201707.050968:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000068bed6b2 118691: 1< 2 - - > 2< 486158336 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70276 - -> 00000080:00200000:2.0:1788201707.050970:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a166d150 00000080:00200000:2.0:1788201707.050970:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.050971:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.050977:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.050984:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.050986:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.050987:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.050993:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.050994:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.050996:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.050998:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051003:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.051020:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.051022:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.051025:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.051027:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118691, 118691] 00000080:00200000:2.0:1788201707.051034:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.051036:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.051054:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.051056:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.051066:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486162432, count: 4096 00000080:00200000:2.0:1788201707.051072:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486162432, bytes: 4096 00000080:00200000:2.0:1788201707.051074:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.051076:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.051077:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486162432, 486166528) 0 0 00000080:00200000:2.0:1788201707.051078:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.051082:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118692, 118692] 00020000:00200000:2.0:1788201707.051084:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486162432 chunk: [486162432, 486166528) 486166528, 4096 00020000:00200000:2.0:1788201707.051085:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.051092:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118692, 118692] 00000008:00010000:2.0:1788201707.051095:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.051096:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.051101:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051107:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486162432 486166527) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051112:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051117:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051121:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.051126:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486162432, 486166528) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.051129:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118692 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.051133:0:17463:0:(rw26.c:898:ll_write_end()) pos 486162432, len 4096, copied 4096 00000080:00200000:2.0:1788201707.051135:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000938d2d37[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118692] 00000080:00200000:2.0:1788201707.051137:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003db0db00 fffffc0001029 2:0 ffff93c0c11afb60 118692 lru 00000080:00200000:2.0:1788201707.051140:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ff6c56c6 118692: 1< 2 - - > 2< 486162432 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70277 - -> 00000080:00200000:2.0:1788201707.051142:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000938d2d37 00000080:00200000:2.0:1788201707.051143:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.051144:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.051149:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.051157:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.051158:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.051160:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.051165:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.051167:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.051168:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.051171:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051175:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.051180:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.051182:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.051184:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.051186:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118692, 118692] 00000080:00200000:2.0:1788201707.051193:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.051194:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.051212:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.051214:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.051224:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486166528, count: 4096 00000080:00200000:2.0:1788201707.051229:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486166528, bytes: 4096 00000080:00200000:2.0:1788201707.051231:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.051234:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.051235:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486166528, 486170624) 0 0 00000080:00200000:2.0:1788201707.051236:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.051239:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118693, 118693] 00020000:00200000:2.0:1788201707.051241:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486166528 chunk: [486166528, 486170624) 486170624, 4096 00020000:00200000:2.0:1788201707.051242:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.051249:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118693, 118693] 00000008:00010000:2.0:1788201707.051252:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.051253:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.051258:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051264:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486166528 486170623) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051269:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051273:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051277:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.051282:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486166528, 486170624) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.051285:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118693 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.051289:0:17463:0:(rw26.c:898:ll_write_end()) pos 486166528, len 4096, copied 4096 00000080:00200000:2.0:1788201707.051291:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e66d3cb2[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118693] 00000080:00200000:2.0:1788201707.051293:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f983f9b8 fffffc0001029 2:0 ffff93c0c11afc40 118693 lru 00000080:00200000:2.0:1788201707.051296:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c3258a5e 118693: 1< 2 - - > 2< 486166528 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70278 - -> 00000080:00200000:2.0:1788201707.051298:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e66d3cb2 00000080:00200000:2.0:1788201707.051298:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.051300:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.051305:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.051312:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.051314:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.051315:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.051321:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.051323:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.051324:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.051327:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051332:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.051336:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.051338:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.051341:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.051342:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118693, 118693] 00000080:00200000:2.0:1788201707.051349:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.051350:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.051368:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.051370:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.051380:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486170624, count: 4096 00000080:00200000:2.0:1788201707.051385:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486170624, bytes: 4096 00000080:00200000:2.0:1788201707.051387:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.051389:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.051390:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486170624, 486174720) 0 0 00000080:00200000:2.0:1788201707.051392:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.051395:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118694, 118694] 00020000:00200000:2.0:1788201707.051397:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486170624 chunk: [486170624, 486174720) 486174720, 4096 00020000:00200000:2.0:1788201707.051398:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.051404:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118694, 118694] 00000008:00010000:2.0:1788201707.051407:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.051409:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.051413:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051419:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486170624 486174719) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051428:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051435:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051442:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.051455:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486170624, 486174720) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.051461:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118694 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.051465:0:17463:0:(rw26.c:898:ll_write_end()) pos 486170624, len 4096, copied 4096 00000080:00200000:2.0:1788201707.051470:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000015806a48[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118694] 00000080:00200000:2.0:1788201707.051472:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f3f73a5a fffffc0001029 2:0 ffff93c0c11af540 118694 lru 00000080:00200000:2.0:1788201707.051476:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002c3e6d2f 118694: 1< 2 - - > 2< 486170624 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70279 - -> 00000080:00200000:2.0:1788201707.051477:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000015806a48 00000080:00200000:2.0:1788201707.051478:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.051479:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.051485:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.051493:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.051495:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.051496:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.051502:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.051504:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.051506:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.051508:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051513:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.051518:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.051519:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.051522:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.051524:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118694, 118694] 00000080:00200000:2.0:1788201707.051531:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.051533:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.051550:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.051552:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.051562:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486174720, count: 4096 00000080:00200000:2.0:1788201707.051568:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486174720, bytes: 4096 00000080:00200000:2.0:1788201707.051570:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.051572:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.051573:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486174720, 486178816) 0 0 00000080:00200000:2.0:1788201707.051574:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.051577:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118695, 118695] 00020000:00200000:2.0:1788201707.051579:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486174720 chunk: [486174720, 486178816) 486178816, 4096 00020000:00200000:2.0:1788201707.051581:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.051587:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118695, 118695] 00000008:00010000:2.0:1788201707.051590:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.051592:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.051597:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051603:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486174720 486178815) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051608:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051612:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051620:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.051624:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486174720, 486178816) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.051627:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118695 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.051630:0:17463:0:(rw26.c:898:ll_write_end()) pos 486174720, len 4096, copied 4096 00000080:00200000:2.0:1788201707.051632:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b38210c5[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118695] 00000080:00200000:2.0:1788201707.051634:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001f782872 fffffc0001029 2:0 ffff93c0c11af460 118695 lru 00000080:00200000:2.0:1788201707.051637:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bd43992e 118695: 1< 2 - - > 2< 486174720 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70280 - -> 00000080:00200000:2.0:1788201707.051639:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b38210c5 00000080:00200000:2.0:1788201707.051639:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.051641:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.051646:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.051653:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.051655:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.051656:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.051662:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.051663:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.051665:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.051667:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051672:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.051680:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.051681:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.051684:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.051686:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118695, 118695] 00000080:00200000:2.0:1788201707.051692:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.051694:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.051711:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.051713:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.051723:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486178816, count: 4096 00000080:00200000:2.0:1788201707.051728:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486178816, bytes: 4096 00000080:00200000:2.0:1788201707.051730:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.051732:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.051733:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486178816, 486182912) 0 0 00000080:00200000:2.0:1788201707.051735:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.051738:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118696, 118696] 00020000:00200000:2.0:1788201707.051740:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486178816 chunk: [486178816, 486182912) 486182912, 4096 00020000:00200000:2.0:1788201707.051741:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.051747:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118696, 118696] 00000008:00010000:2.0:1788201707.051750:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.051752:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.051757:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051762:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486178816 486182911) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051768:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051776:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051780:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.051784:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486178816, 486182912) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.051787:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118696 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.051790:0:17463:0:(rw26.c:898:ll_write_end()) pos 486178816, len 4096, copied 4096 00000080:00200000:2.0:1788201707.051792:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bdaa151f[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118696] 00000080:00200000:2.0:1788201707.051794:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004d8fa2d1 fffffc0001029 2:0 ffff93c0c11af0e0 118696 lru 00000080:00200000:2.0:1788201707.051797:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b44bb2af 118696: 1< 2 - - > 2< 486178816 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70281 - -> 00000080:00200000:2.0:1788201707.051798:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bdaa151f 00000080:00200000:2.0:1788201707.051799:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.051800:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.051805:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.051813:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.051814:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.051815:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.051821:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.051823:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.051824:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.051826:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051851:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.051856:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.051858:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.051861:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.051863:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118696, 118696] 00000080:00200000:2.0:1788201707.051872:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.051874:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.051892:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.051894:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.051903:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486182912, count: 4096 00000080:00200000:2.0:1788201707.051909:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486182912, bytes: 4096 00000080:00200000:2.0:1788201707.051911:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.051913:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.051914:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486182912, 486187008) 0 0 00000080:00200000:2.0:1788201707.051915:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.051919:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118697, 118697] 00020000:00200000:2.0:1788201707.051920:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486182912 chunk: [486182912, 486187008) 486187008, 4096 00020000:00200000:2.0:1788201707.051922:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.051928:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118697, 118697] 00000008:00010000:2.0:1788201707.051931:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.051933:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.051937:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051943:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486182912 486187007) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051952:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051957:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.051961:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.051964:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486182912, 486187008) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.051967:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118697 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.051971:0:17463:0:(rw26.c:898:ll_write_end()) pos 486182912, len 4096, copied 4096 00000080:00200000:2.0:1788201707.051973:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001ce5e9e6[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118697] 00000080:00200000:2.0:1788201707.051975:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b745e4c8 fffffc0001029 2:0 ffff93c0c11af700 118697 lru 00000080:00200000:2.0:1788201707.051978:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e2e37dd5 118697: 1< 2 - - > 2< 486182912 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70282 - -> 00000080:00200000:2.0:1788201707.051979:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001ce5e9e6 00000080:00200000:2.0:1788201707.051980:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.051981:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.051986:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.051994:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.051996:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.051997:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.052003:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.052004:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.052006:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.052021:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052027:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.052032:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.052033:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.052036:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.052038:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118697, 118697] 00000080:00200000:2.0:1788201707.052045:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.052047:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.052065:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.052068:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.052077:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486187008, count: 4096 00000080:00200000:2.0:1788201707.052083:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486187008, bytes: 4096 00000080:00200000:2.0:1788201707.052085:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.052087:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.052088:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486187008, 486191104) 0 0 00000080:00200000:2.0:1788201707.052090:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.052093:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118698, 118698] 00020000:00200000:2.0:1788201707.052095:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486187008 chunk: [486187008, 486191104) 486191104, 4096 00020000:00200000:2.0:1788201707.052096:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.052103:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118698, 118698] 00000008:00010000:2.0:1788201707.052106:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.052107:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.052112:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052121:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486187008 486191103) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052126:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052131:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052134:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.052138:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486187008, 486191104) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.052141:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118698 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.052145:0:17463:0:(rw26.c:898:ll_write_end()) pos 486187008, len 4096, copied 4096 00000080:00200000:2.0:1788201707.052147:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000dc38839c[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118698] 00000080:00200000:2.0:1788201707.052149:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bd76feff fffffc0001029 2:0 ffff93c0c11af000 118698 lru 00000080:00200000:2.0:1788201707.052152:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008dbdc8d9 118698: 1< 2 - - > 2< 486187008 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70283 - -> 00000080:00200000:2.0:1788201707.052154:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000dc38839c 00000080:00200000:2.0:1788201707.052154:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.052155:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.052161:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.052168:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.052170:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.052171:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.052177:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.052179:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.052180:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.052183:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052187:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.052192:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.052194:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.052196:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.052198:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118698, 118698] 00000080:00200000:2.0:1788201707.052204:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.052206:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.052223:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.052226:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.052235:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486191104, count: 4096 00000080:00200000:2.0:1788201707.052241:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486191104, bytes: 4096 00000080:00200000:2.0:1788201707.052242:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.052245:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.052246:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486191104, 486195200) 0 0 00000080:00200000:2.0:1788201707.052247:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.052250:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118699, 118699] 00020000:00200000:2.0:1788201707.052252:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486191104 chunk: [486191104, 486195200) 486195200, 4096 00020000:00200000:2.0:1788201707.052254:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.052260:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118699, 118699] 00000008:00010000:2.0:1788201707.052263:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.052264:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.052270:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052275:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486191104 486195199) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052281:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052285:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052289:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.052294:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486191104, 486195200) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.052297:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118699 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.052300:0:17463:0:(rw26.c:898:ll_write_end()) pos 486191104, len 4096, copied 4096 00000080:00200000:2.0:1788201707.052303:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c89a541a[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118699] 00000080:00200000:2.0:1788201707.052304:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000077ea5546 fffffc0001029 2:0 ffff93c0c13b89a0 118699 lru 00000080:00200000:2.0:1788201707.052308:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e5884e77 118699: 1< 2 - - > 2< 486191104 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70284 - -> 00000080:00200000:2.0:1788201707.052309:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c89a541a 00000080:00200000:2.0:1788201707.052310:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.052311:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.052316:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.052324:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.052326:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.052327:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.052333:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.052335:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.052337:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.052339:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052344:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.052349:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.052351:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.052353:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.052355:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118699, 118699] 00000080:00200000:2.0:1788201707.052361:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.052363:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.052380:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.052382:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.052392:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486195200, count: 4096 00000080:00200000:2.0:1788201707.052398:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486195200, bytes: 4096 00000080:00200000:2.0:1788201707.052399:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.052401:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.052403:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486195200, 486199296) 0 0 00000080:00200000:2.0:1788201707.052404:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.052407:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118700, 118700] 00020000:00200000:2.0:1788201707.052409:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486195200 chunk: [486195200, 486199296) 486199296, 4096 00020000:00200000:2.0:1788201707.052410:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.052417:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118700, 118700] 00000008:00010000:2.0:1788201707.052421:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.052423:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.052430:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052439:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486195200 486199295) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052447:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052455:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052463:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.052471:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486195200, 486199296) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.052476:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118700 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.052481:0:17463:0:(rw26.c:898:ll_write_end()) pos 486195200, len 4096, copied 4096 00000080:00200000:2.0:1788201707.052484:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006f690008[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118700] 00000080:00200000:2.0:1788201707.052486:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000010f0b0f8 fffffc0001029 2:0 ffff93c0c13b8b60 118700 lru 00000080:00200000:2.0:1788201707.052489:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000048d9d9d5 118700: 1< 2 - - > 2< 486195200 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70285 - -> 00000080:00200000:2.0:1788201707.052491:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006f690008 00000080:00200000:2.0:1788201707.052492:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.052493:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.052499:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.052510:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.052511:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.052513:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.052519:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.052521:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.052522:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.052525:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052530:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.052535:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.052536:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.052538:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.052540:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118700, 118700] 00000080:00200000:2.0:1788201707.052547:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.052549:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.052566:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.052568:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.052578:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486199296, count: 4096 00000080:00200000:2.0:1788201707.052584:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486199296, bytes: 4096 00000080:00200000:2.0:1788201707.052586:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.052588:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.052589:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486199296, 486203392) 0 0 00000080:00200000:2.0:1788201707.052591:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.052594:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118701, 118701] 00020000:00200000:2.0:1788201707.052596:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486199296 chunk: [486199296, 486203392) 486203392, 4096 00020000:00200000:2.0:1788201707.052598:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.052604:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118701, 118701] 00000008:00010000:2.0:1788201707.052607:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.052609:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.052614:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052620:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486199296 486203391) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052625:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052629:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052633:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.052638:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486199296, 486203392) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.052641:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118701 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.052645:0:17463:0:(rw26.c:898:ll_write_end()) pos 486199296, len 4096, copied 4096 00000080:00200000:2.0:1788201707.052647:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000084759a4e[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118701] 00000080:00200000:2.0:1788201707.052649:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009437d0ba fffffc0001029 2:0 ffff93c0c13b8c40 118701 lru 00000080:00200000:2.0:1788201707.052652:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007e2036fe 118701: 1< 2 - - > 2< 486199296 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70286 - -> 00000080:00200000:2.0:1788201707.052654:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000084759a4e 00000080:00200000:2.0:1788201707.052655:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.052656:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.052661:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.052669:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.052671:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.052672:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.052678:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.052679:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.052681:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.052683:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052687:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.052692:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.052693:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.052696:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.052698:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118701, 118701] 00000080:00200000:2.0:1788201707.052704:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.052706:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.052723:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.052725:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.052734:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486203392, count: 4096 00000080:00200000:2.0:1788201707.052740:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486203392, bytes: 4096 00000080:00200000:2.0:1788201707.052741:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.052744:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.052745:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486203392, 486207488) 0 0 00000080:00200000:2.0:1788201707.052746:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.052750:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118702, 118702] 00020000:00200000:2.0:1788201707.052751:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486203392 chunk: [486203392, 486207488) 486207488, 4096 00020000:00200000:2.0:1788201707.052753:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.052759:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118702, 118702] 00000008:00010000:2.0:1788201707.052762:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.052763:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.052768:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052774:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486203392 486207487) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052779:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052784:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052788:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.052792:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486203392, 486207488) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.052795:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118702 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.052798:0:17463:0:(rw26.c:898:ll_write_end()) pos 486203392, len 4096, copied 4096 00000080:00200000:2.0:1788201707.052801:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004c18e245[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118702] 00000080:00200000:2.0:1788201707.052802:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000eada1565 fffffc0001029 2:0 ffff93c0c13b8540 118702 lru 00000080:00200000:2.0:1788201707.052806:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000147f4baf 118702: 1< 2 - - > 2< 486203392 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70287 - -> 00000080:00200000:2.0:1788201707.052808:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004c18e245 00000080:00200000:2.0:1788201707.052809:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.052810:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.052815:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.052823:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.052824:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.052826:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.052843:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.052845:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.052847:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.052849:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052854:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.052859:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.052860:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.052863:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.052865:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118702, 118702] 00000080:00200000:2.0:1788201707.052871:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.052873:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.052891:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.052893:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.052903:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486207488, count: 4096 00000080:00200000:2.0:1788201707.052908:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486207488, bytes: 4096 00000080:00200000:2.0:1788201707.052910:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.052913:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.052914:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486207488, 486211584) 0 0 00000080:00200000:2.0:1788201707.052915:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.052918:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118703, 118703] 00020000:00200000:2.0:1788201707.052920:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486207488 chunk: [486207488, 486211584) 486211584, 4096 00020000:00200000:2.0:1788201707.052921:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.052928:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118703, 118703] 00000008:00010000:2.0:1788201707.052931:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.052932:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.052937:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052942:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486207488 486211583) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052948:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052952:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.052956:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.052960:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486207488, 486211584) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.052963:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118703 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.052966:0:17463:0:(rw26.c:898:ll_write_end()) pos 486207488, len 4096, copied 4096 00000080:00200000:2.0:1788201707.052968:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f40c51c2[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118703] 00000080:00200000:2.0:1788201707.052970:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f37f04b8 fffffc0001029 2:0 ffff93c0c13b8460 118703 lru 00000080:00200000:2.0:1788201707.052974:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d6000e18 118703: 1< 2 - - > 2< 486207488 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70288 - -> 00000080:00200000:2.0:1788201707.052975:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f40c51c2 00000080:00200000:2.0:1788201707.052976:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.052978:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.052983:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.052990:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.052991:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.052992:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.052998:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.053000:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.053002:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.053004:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053021:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.053026:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.053028:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.053031:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.053033:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118703, 118703] 00000080:00200000:2.0:1788201707.053040:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.053042:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.053060:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.053062:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.053073:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486211584, count: 4096 00000080:00200000:2.0:1788201707.053078:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486211584, bytes: 4096 00000080:00200000:2.0:1788201707.053081:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.053083:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.053084:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486211584, 486215680) 0 0 00000080:00200000:2.0:1788201707.053085:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.053089:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118704, 118704] 00020000:00200000:2.0:1788201707.053090:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486211584 chunk: [486211584, 486215680) 486215680, 4096 00020000:00200000:2.0:1788201707.053092:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.053098:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118704, 118704] 00000008:00010000:2.0:1788201707.053101:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.053103:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.053108:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053114:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486211584 486215679) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053119:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053124:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053128:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.053133:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486211584, 486215680) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.053136:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118704 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.053140:0:17463:0:(rw26.c:898:ll_write_end()) pos 486211584, len 4096, copied 4096 00000080:00200000:2.0:1788201707.053142:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000028fafba0[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118704] 00000080:00200000:2.0:1788201707.053144:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bde676f1 fffffc0001029 2:0 ffff93c0c13b80e0 118704 lru 00000080:00200000:2.0:1788201707.053147:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008bb211c3 118704: 1< 2 - - > 2< 486211584 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70289 - -> 00000080:00200000:2.0:1788201707.053149:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000028fafba0 00000080:00200000:2.0:1788201707.053149:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.053151:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.053156:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.053163:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.053165:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.053166:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.053172:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.053174:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.053175:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.053178:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053182:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.053187:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.053188:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.053191:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.053193:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118704, 118704] 00000080:00200000:2.0:1788201707.053199:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.053201:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.053218:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.053220:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.053230:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486215680, count: 4096 00000080:00200000:2.0:1788201707.053235:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486215680, bytes: 4096 00000080:00200000:2.0:1788201707.053237:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.053240:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.053241:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486215680, 486219776) 0 0 00000080:00200000:2.0:1788201707.053242:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.053245:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118705, 118705] 00020000:00200000:2.0:1788201707.053247:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486215680 chunk: [486215680, 486219776) 486219776, 4096 00020000:00200000:2.0:1788201707.053248:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.053255:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118705, 118705] 00000008:00010000:2.0:1788201707.053258:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.053259:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.053264:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053270:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486215680 486219775) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053275:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053279:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053283:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.053288:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486215680, 486219776) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.053291:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118705 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.053295:0:17463:0:(rw26.c:898:ll_write_end()) pos 486215680, len 4096, copied 4096 00000080:00200000:2.0:1788201707.053297:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f55e3eb8[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118705] 00000080:00200000:2.0:1788201707.053299:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fce5ac7d fffffc0001029 2:0 ffff93c0c13b8700 118705 lru 00000080:00200000:2.0:1788201707.053303:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001ef654e0 118705: 1< 2 - - > 2< 486215680 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70290 - -> 00000080:00200000:2.0:1788201707.053304:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f55e3eb8 00000080:00200000:2.0:1788201707.053305:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.053306:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.053311:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.053318:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.053320:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.053321:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.053327:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.053329:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.053330:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.053332:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053337:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.053341:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.053343:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.053345:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.053347:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118705, 118705] 00000080:00200000:2.0:1788201707.053354:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.053355:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.053373:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.053375:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.053386:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486219776, count: 4096 00000080:00200000:2.0:1788201707.053391:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486219776, bytes: 4096 00000080:00200000:2.0:1788201707.053393:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.053395:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.053396:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486219776, 486223872) 0 0 00000080:00200000:2.0:1788201707.053398:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.053401:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118706, 118706] 00020000:00200000:2.0:1788201707.053403:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486219776 chunk: [486219776, 486223872) 486223872, 4096 00020000:00200000:2.0:1788201707.053404:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.053411:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118706, 118706] 00000008:00010000:2.0:1788201707.053414:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.053415:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.053421:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053429:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486219776 486223871) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053438:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053445:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053452:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.053466:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486219776, 486223872) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.053472:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118706 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.053475:0:17463:0:(rw26.c:898:ll_write_end()) pos 486219776, len 4096, copied 4096 00000080:00200000:2.0:1788201707.053478:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f1627913[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118706] 00000080:00200000:2.0:1788201707.053480:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000163026dd fffffc0001029 2:0 ffff93c0c13b8000 118706 lru 00000080:00200000:2.0:1788201707.053483:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000034efc90c 118706: 1< 2 - - > 2< 486219776 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70291 - -> 00000080:00200000:2.0:1788201707.053485:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f1627913 00000080:00200000:2.0:1788201707.053485:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.053487:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.053492:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.053500:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.053502:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.053504:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.053510:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.053511:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.053513:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.053515:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053520:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.053525:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.053526:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.053529:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.053531:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118706, 118706] 00000080:00200000:2.0:1788201707.053538:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.053540:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.053557:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.053559:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.053569:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486223872, count: 4096 00000080:00200000:2.0:1788201707.053575:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486223872, bytes: 4096 00000080:00200000:2.0:1788201707.053577:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.053579:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.053580:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486223872, 486227968) 0 0 00000080:00200000:2.0:1788201707.053581:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.053585:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118707, 118707] 00020000:00200000:2.0:1788201707.053586:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486223872 chunk: [486223872, 486227968) 486227968, 4096 00020000:00200000:2.0:1788201707.053588:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.053594:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118707, 118707] 00000008:00010000:2.0:1788201707.053597:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.053599:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.053604:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053609:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486223872 486227967) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053614:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053619:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053626:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.053630:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486223872, 486227968) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.053633:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118707 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.053637:0:17463:0:(rw26.c:898:ll_write_end()) pos 486223872, len 4096, copied 4096 00000080:00200000:2.0:1788201707.053639:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007cd3dcbf[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118707] 00000080:00200000:2.0:1788201707.053641:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000877a3ef2 fffffc0001029 2:0 ffff93c0c13b8a80 118707 lru 00000080:00200000:2.0:1788201707.053644:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000e77fb5a 118707: 1< 2 - - > 2< 486223872 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70292 - -> 00000080:00200000:2.0:1788201707.053646:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007cd3dcbf 00000080:00200000:2.0:1788201707.053646:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.053648:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.053653:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.053660:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.053662:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.053663:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.053669:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.053671:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.053672:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.053675:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053679:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.053687:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.053689:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.053691:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.053693:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118707, 118707] 00000080:00200000:2.0:1788201707.053700:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.053701:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.053718:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.053721:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.053730:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486227968, count: 4096 00000080:00200000:2.0:1788201707.053736:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486227968, bytes: 4096 00000080:00200000:2.0:1788201707.053737:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.053740:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.053741:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486227968, 486232064) 0 0 00000080:00200000:2.0:1788201707.053742:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.053745:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118708, 118708] 00020000:00200000:2.0:1788201707.053747:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486227968 chunk: [486227968, 486232064) 486232064, 4096 00020000:00200000:2.0:1788201707.053748:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.053755:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118708, 118708] 00000008:00010000:2.0:1788201707.053758:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.053759:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.053764:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053769:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486227968 486232063) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053775:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053783:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053787:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.053791:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486227968, 486232064) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.053794:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118708 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.053797:0:17463:0:(rw26.c:898:ll_write_end()) pos 486227968, len 4096, copied 4096 00000080:00200000:2.0:1788201707.053800:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003238af46[3 0000000075f4bf38 2 1 00000000d653e89e 0 0 118708] 00000080:00200000:2.0:1788201707.053801:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e2064849 fffffc0001029 2:0 ffff93c0c13b8380 118708 lru 00000080:00200000:2.0:1788201707.053805:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b683a32e 118708: 1< 2 - - > 2< 486227968 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70293 - -> 00000080:00200000:2.0:1788201707.053806:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003238af46 00000080:00200000:2.0:1788201707.053806:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.053807:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.053812:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.053820:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.053821:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.053823:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.053843:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.053845:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.053847:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.053849:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.053858:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.053863:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57028a8 (ffff93c1d300a638/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.053865:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a638/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.053868:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.053870:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118708, 118708] 00000080:00200000:2.0:1788201707.053877:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.053879:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.053925:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.053930:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.053951:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486232064, count: 4096 00000080:00200000:3.0:1788201707.053966:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486232064, bytes: 4096 00000080:00200000:3.0:1788201707.053969:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.053972:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.053973:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486232064, 486236160) 0 0 00000080:00200000:3.0:1788201707.053975:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.053985:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118709, 118709] 00020000:00200000:3.0:1788201707.053989:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486232064 chunk: [486232064, 486236160) 486236160, 4096 00020000:00200000:3.0:1788201707.053990:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.054004:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118709, 118709] 00000008:00010000:3.0:1788201707.054019:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.054022:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.054034:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054042:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486232064 486236159) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054049:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054054:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054058:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.054063:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486232064, 486236160) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.054068:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118709 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.054073:0:17463:0:(rw26.c:898:ll_write_end()) pos 486232064, len 4096, copied 4096 00000080:00200000:3.0:1788201707.054076:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002e82471e[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118709] 00000080:00200000:3.0:1788201707.054078:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000046c4a427 fffffc0001029 2:0 ffff93c0c13b88c0 118709 lru 00000080:00200000:3.0:1788201707.054082:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003ee365a9 118709: 1< 2 - - > 2< 486232064 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70294 - -> 00000080:00200000:3.0:1788201707.054083:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002e82471e 00000080:00200000:3.0:1788201707.054084:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.054085:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.054092:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.054102:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.054104:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.054105:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.054114:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.054117:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.054118:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.054121:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054126:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.054132:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.054133:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.054136:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.054138:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118709, 118709] 00000080:00200000:3.0:1788201707.054153:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.054155:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.054178:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.054181:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.054200:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486236160, count: 4096 00000080:00200000:3.0:1788201707.054207:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486236160, bytes: 4096 00000080:00200000:3.0:1788201707.054209:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.054212:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.054213:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486236160, 486240256) 0 0 00000080:00200000:3.0:1788201707.054215:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.054218:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118710, 118710] 00020000:00200000:3.0:1788201707.054220:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486236160 chunk: [486236160, 486240256) 486240256, 4096 00020000:00200000:3.0:1788201707.054222:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.054230:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118710, 118710] 00000008:00010000:3.0:1788201707.054233:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.054235:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.054241:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054247:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486236160 486240255) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054253:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054257:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054261:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.054266:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486236160, 486240256) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.054270:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118710 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.054284:0:17463:0:(rw26.c:898:ll_write_end()) pos 486236160, len 4096, copied 4096 00000080:00200000:3.0:1788201707.054287:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002be51ba3[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118710] 00000080:00200000:3.0:1788201707.054289:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c5e3ef18 fffffc0001029 2:0 ffff93c0c13b8ee0 118710 lru 00000080:00200000:3.0:1788201707.054293:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000033573c0c 118710: 1< 2 - - > 2< 486236160 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70295 - -> 00000080:00200000:3.0:1788201707.054295:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002be51ba3 00000080:00200000:3.0:1788201707.054295:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.054297:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.054303:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.054311:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.054313:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.054314:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.054321:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.054323:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.054325:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.054327:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054332:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.054337:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.054338:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.054340:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.054342:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118710, 118710] 00000080:00200000:3.0:1788201707.054357:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.054360:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.054379:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.054381:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.054392:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486240256, count: 4096 00000080:00200000:3.0:1788201707.054398:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486240256, bytes: 4096 00000080:00200000:3.0:1788201707.054400:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.054403:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.054404:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486240256, 486244352) 0 0 00000080:00200000:3.0:1788201707.054405:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.054409:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118711, 118711] 00020000:00200000:3.0:1788201707.054411:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486240256 chunk: [486240256, 486244352) 486244352, 4096 00020000:00200000:3.0:1788201707.054412:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.054419:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118711, 118711] 00000008:00010000:3.0:1788201707.054423:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.054424:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.054430:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054435:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486240256 486244351) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054441:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054445:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054449:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.054457:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486240256, 486244352) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.054461:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118711 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.054465:0:17463:0:(rw26.c:898:ll_write_end()) pos 486240256, len 4096, copied 4096 00000080:00200000:3.0:1788201707.054467:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f93f8257[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118711] 00000080:00200000:3.0:1788201707.054469:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d5c928da fffffc0001029 2:0 ffff93c0c13b87e0 118711 lru 00000080:00200000:3.0:1788201707.054473:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000093063f06 118711: 1< 2 - - > 2< 486240256 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70296 - -> 00000080:00200000:3.0:1788201707.054475:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f93f8257 00000080:00200000:3.0:1788201707.054475:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.054477:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.054483:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.054491:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.054492:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.054494:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.054500:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.054502:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.054503:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.054506:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054510:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.054515:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.054517:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.054519:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.054521:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118711, 118711] 00000080:00200000:3.0:1788201707.054529:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.054531:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.054550:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.054552:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.054562:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486244352, count: 4096 00000080:00200000:3.0:1788201707.054568:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486244352, bytes: 4096 00000080:00200000:3.0:1788201707.054569:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.054572:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.054573:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486244352, 486248448) 0 0 00000080:00200000:3.0:1788201707.054575:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.054578:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118712, 118712] 00020000:00200000:3.0:1788201707.054580:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486244352 chunk: [486244352, 486248448) 486248448, 4096 00020000:00200000:3.0:1788201707.054581:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.054588:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118712, 118712] 00000008:00010000:3.0:1788201707.054591:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.054592:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.054597:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054603:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486244352 486248447) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054608:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054613:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054620:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.054625:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486244352, 486248448) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.054636:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118712 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.054640:0:17463:0:(rw26.c:898:ll_write_end()) pos 486244352, len 4096, copied 4096 00000080:00200000:3.0:1788201707.054642:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000050ae83a[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118712] 00000080:00200000:3.0:1788201707.054644:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007ff85b07 fffffc0001029 2:0 ffff93c0c13b82a0 118712 lru 00000080:00200000:3.0:1788201707.054648:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ed02cf66 118712: 1< 2 - - > 2< 486244352 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70297 - -> 00000080:00200000:3.0:1788201707.054649:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000050ae83a 00000080:00200000:3.0:1788201707.054650:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.054652:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.054657:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.054665:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.054667:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.054668:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.054674:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.054676:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.054678:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.054680:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054685:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.054694:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.054695:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.054698:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.054700:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118712, 118712] 00000080:00200000:3.0:1788201707.054707:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.054709:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.054727:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.054730:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.054740:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486248448, count: 4096 00000080:00200000:3.0:1788201707.054746:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486248448, bytes: 4096 00000080:00200000:3.0:1788201707.054747:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.054750:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.054751:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486248448, 486252544) 0 0 00000080:00200000:3.0:1788201707.054752:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.054756:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118713, 118713] 00020000:00200000:3.0:1788201707.054757:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486248448 chunk: [486248448, 486252544) 486252544, 4096 00020000:00200000:3.0:1788201707.054759:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.054766:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118713, 118713] 00000008:00010000:3.0:1788201707.054769:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.054770:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.054775:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054781:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486248448 486252543) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054787:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054795:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054799:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.054803:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486248448, 486252544) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.054806:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118713 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.054810:0:17463:0:(rw26.c:898:ll_write_end()) pos 486248448, len 4096, copied 4096 00000080:00200000:3.0:1788201707.054812:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000df682b01[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118713] 00000080:00200000:3.0:1788201707.054814:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002dc56b3d fffffc0001029 2:0 ffff93c0c13b8e00 118713 lru 00000080:00200000:3.0:1788201707.054817:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cdb5f72f 118713: 1< 2 - - > 2< 486248448 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70298 - -> 00000080:00200000:3.0:1788201707.054818:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000df682b01 00000080:00200000:3.0:1788201707.054819:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.054820:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.054826:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.054852:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.054853:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.054855:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.054861:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.054863:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.054864:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.054867:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054875:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.054880:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.054882:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.054884:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.054886:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118713, 118713] 00000080:00200000:3.0:1788201707.054894:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.054895:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.054914:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.054916:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.054926:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486252544, count: 4096 00000080:00200000:3.0:1788201707.054932:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486252544, bytes: 4096 00000080:00200000:3.0:1788201707.054934:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.054937:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.054938:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486252544, 486256640) 0 0 00000080:00200000:3.0:1788201707.054939:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.054942:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118714, 118714] 00020000:00200000:3.0:1788201707.054944:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486252544 chunk: [486252544, 486256640) 486256640, 4096 00020000:00200000:3.0:1788201707.054946:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.054953:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118714, 118714] 00000008:00010000:3.0:1788201707.054956:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.054957:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.054962:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054968:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486252544 486256639) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054977:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054982:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.054986:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.054990:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486252544, 486256640) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.054993:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118714 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.054997:0:17463:0:(rw26.c:898:ll_write_end()) pos 486252544, len 4096, copied 4096 00000080:00200000:3.0:1788201707.055000:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d2e4f8fa[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118714] 00000080:00200000:3.0:1788201707.055002:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000030382c4a fffffc0001029 2:0 ffff93c0c13b81c0 118714 lru 00000080:00200000:3.0:1788201707.055005:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000071e1329 118714: 1< 2 - - > 2< 486252544 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70299 - -> 00000080:00200000:3.0:1788201707.055018:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d2e4f8fa 00000080:00200000:3.0:1788201707.055019:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.055021:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.055027:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.055035:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.055037:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.055038:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.055044:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.055046:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.055048:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.055051:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055056:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.055061:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.055062:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.055064:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.055066:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118714, 118714] 00000080:00200000:3.0:1788201707.055074:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.055076:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.055094:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.055096:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.055106:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486256640, count: 4096 00000080:00200000:3.0:1788201707.055112:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486256640, bytes: 4096 00000080:00200000:3.0:1788201707.055114:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.055116:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.055117:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486256640, 486260736) 0 0 00000080:00200000:3.0:1788201707.055119:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.055122:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118715, 118715] 00020000:00200000:3.0:1788201707.055124:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486256640 chunk: [486256640, 486260736) 486260736, 4096 00020000:00200000:3.0:1788201707.055125:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.055132:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118715, 118715] 00000008:00010000:3.0:1788201707.055135:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.055137:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.055142:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055151:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486256640 486260735) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055156:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055161:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055165:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.055169:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486256640, 486260736) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.055172:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118715 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.055175:0:17463:0:(rw26.c:898:ll_write_end()) pos 486256640, len 4096, copied 4096 00000080:00200000:3.0:1788201707.055177:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000439be56a[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118715] 00000080:00200000:3.0:1788201707.055179:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008f62eb9a fffffc0001029 2:0 ffff93c0c13b8620 118715 lru 00000080:00200000:3.0:1788201707.055183:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000758967e4 118715: 1< 2 - - > 2< 486256640 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70300 - -> 00000080:00200000:3.0:1788201707.055184:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000439be56a 00000080:00200000:3.0:1788201707.055185:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.055186:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.055191:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.055198:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.055200:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.055202:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.055207:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.055209:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.055211:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.055213:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055217:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.055222:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.055223:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.055226:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.055228:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118715, 118715] 00000080:00200000:3.0:1788201707.055235:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.055237:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.055255:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.055257:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.055267:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486260736, count: 4096 00000080:00200000:3.0:1788201707.055273:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486260736, bytes: 4096 00000080:00200000:3.0:1788201707.055275:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.055277:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.055278:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486260736, 486264832) 0 0 00000080:00200000:3.0:1788201707.055279:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.055283:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118716, 118716] 00020000:00200000:3.0:1788201707.055284:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486260736 chunk: [486260736, 486264832) 486264832, 4096 00020000:00200000:3.0:1788201707.055286:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.055293:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118716, 118716] 00000008:00010000:3.0:1788201707.055296:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.055297:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.055302:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055308:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486260736 486264831) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055314:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055318:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055322:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.055326:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486260736, 486264832) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.055330:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118716 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.055333:0:17463:0:(rw26.c:898:ll_write_end()) pos 486260736, len 4096, copied 4096 00000080:00200000:3.0:1788201707.055336:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000cbbbdf25[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118716] 00000080:00200000:3.0:1788201707.055338:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000001473cc6 fffffc0001029 2:0 ffff93c0c13b8d20 118716 lru 00000080:00200000:3.0:1788201707.055341:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000959902f7 118716: 1< 2 - - > 2< 486260736 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70301 - -> 00000080:00200000:3.0:1788201707.055343:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000cbbbdf25 00000080:00200000:3.0:1788201707.055343:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.055345:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.055350:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.055357:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.055359:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.055361:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.055366:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.055368:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.055369:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.055372:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055376:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.055381:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.055383:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.055385:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.055386:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118716, 118716] 00000080:00200000:3.0:1788201707.055394:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.055396:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.055413:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.055415:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.055425:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486264832, count: 4096 00000080:00200000:3.0:1788201707.055431:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486264832, bytes: 4096 00000080:00200000:3.0:1788201707.055433:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.055435:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.055436:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486264832, 486268928) 0 0 00000080:00200000:3.0:1788201707.055438:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.055441:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118717, 118717] 00020000:00200000:3.0:1788201707.055443:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486264832 chunk: [486264832, 486268928) 486268928, 4096 00020000:00200000:3.0:1788201707.055444:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.055451:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118717, 118717] 00000008:00010000:3.0:1788201707.055454:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.055456:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.055460:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055466:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486264832 486268927) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055471:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055476:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055479:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.055483:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486264832, 486268928) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.055486:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118717 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.055490:0:17463:0:(rw26.c:898:ll_write_end()) pos 486264832, len 4096, copied 4096 00000080:00200000:3.0:1788201707.055492:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c0c1c6c5[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118717] 00000080:00200000:3.0:1788201707.055494:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000036a85ab7 fffffc0001029 2:0 ffff93c0c13b9b60 118717 lru 00000080:00200000:3.0:1788201707.055497:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f8cc00b9 118717: 1< 2 - - > 2< 486264832 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70302 - -> 00000080:00200000:3.0:1788201707.055499:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c0c1c6c5 00000080:00200000:3.0:1788201707.055500:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.055501:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.055506:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.055516:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.055518:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.055519:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.055525:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.055527:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.055529:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.055531:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055536:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.055541:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.055542:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.055544:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.055546:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118717, 118717] 00000080:00200000:3.0:1788201707.055554:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.055556:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.055573:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.055576:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.055586:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486268928, count: 4096 00000080:00200000:3.0:1788201707.055592:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486268928, bytes: 4096 00000080:00200000:3.0:1788201707.055593:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.055596:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.055597:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486268928, 486273024) 0 0 00000080:00200000:3.0:1788201707.055598:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.055601:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118718, 118718] 00020000:00200000:3.0:1788201707.055604:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486268928 chunk: [486268928, 486273024) 486273024, 4096 00020000:00200000:3.0:1788201707.055605:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.055612:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118718, 118718] 00000008:00010000:3.0:1788201707.055615:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.055617:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.055621:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055627:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486268928 486273023) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055632:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055637:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055640:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.055645:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486268928, 486273024) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.055648:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118718 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.055651:0:17463:0:(rw26.c:898:ll_write_end()) pos 486268928, len 4096, copied 4096 00000080:00200000:3.0:1788201707.055653:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000acfb50ef[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118718] 00000080:00200000:3.0:1788201707.055656:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d33875ab fffffc0001029 2:0 ffff93c0c13b9c40 118718 lru 00000080:00200000:3.0:1788201707.055660:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e4315606 118718: 1< 2 - - > 2< 486268928 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70303 - -> 00000080:00200000:3.0:1788201707.055661:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000acfb50ef 00000080:00200000:3.0:1788201707.055662:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.055663:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.055669:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.055676:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.055678:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.055679:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.055685:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.055686:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.055688:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.055690:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055694:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.055699:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.055700:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.055702:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.055704:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118718, 118718] 00000080:00200000:3.0:1788201707.055712:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.055713:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.055731:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.055733:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.055743:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486273024, count: 4096 00000080:00200000:3.0:1788201707.055749:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486273024, bytes: 4096 00000080:00200000:3.0:1788201707.055751:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.055753:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.055754:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486273024, 486277120) 0 0 00000080:00200000:3.0:1788201707.055756:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.055759:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118719, 118719] 00020000:00200000:3.0:1788201707.055761:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486273024 chunk: [486273024, 486277120) 486277120, 4096 00020000:00200000:3.0:1788201707.055762:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.055769:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118719, 118719] 00000008:00010000:3.0:1788201707.055772:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.055773:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.055778:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055784:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486273024 486277119) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055789:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055794:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055798:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.055802:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486273024, 486277120) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.055805:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118719 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.055808:0:17463:0:(rw26.c:898:ll_write_end()) pos 486273024, len 4096, copied 4096 00000080:00200000:3.0:1788201707.055810:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001eb3ac2e[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118719] 00000080:00200000:3.0:1788201707.055812:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a65d07e9 fffffc0001029 2:0 ffff93c0c13b9540 118719 lru 00000080:00200000:3.0:1788201707.055815:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006210e5e2 118719: 1< 2 - - > 2< 486273024 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70304 - -> 00000080:00200000:3.0:1788201707.055818:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001eb3ac2e 00000080:00200000:3.0:1788201707.055819:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.055820:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.055825:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.055848:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.055850:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.055851:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.055857:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.055858:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.055860:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.055862:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055867:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.055872:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.055873:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.055875:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.055877:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118719, 118719] 00000080:00200000:3.0:1788201707.055885:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.055887:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.055904:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.055906:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.055916:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486277120, count: 4096 00000080:00200000:3.0:1788201707.055922:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486277120, bytes: 4096 00000080:00200000:3.0:1788201707.055924:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.055926:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.055927:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486277120, 486281216) 0 0 00000080:00200000:3.0:1788201707.055929:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.055932:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118720, 118720] 00020000:00200000:3.0:1788201707.055934:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486277120 chunk: [486277120, 486281216) 486281216, 4096 00020000:00200000:3.0:1788201707.055935:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.055942:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118720, 118720] 00000008:00010000:3.0:1788201707.055945:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.055946:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.055951:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055957:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486277120 486281215) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055962:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055966:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.055970:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.055975:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486277120, 486281216) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.055978:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118720 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.055981:0:17463:0:(rw26.c:898:ll_write_end()) pos 486277120, len 4096, copied 4096 00000080:00200000:3.0:1788201707.055983:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000185e74e1[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118720] 00000080:00200000:3.0:1788201707.055985:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007bdf50f2 fffffc0001029 2:0 ffff93c0c13b9460 118720 lru 00000080:00200000:3.0:1788201707.055989:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002706ce23 118720: 1< 2 - - > 2< 486277120 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70305 - -> 00000080:00200000:3.0:1788201707.055990:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000185e74e1 00000080:00200000:3.0:1788201707.055991:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.055992:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.055998:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.056020:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.056023:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.056025:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.056036:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.056039:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.056041:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.056045:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056052:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.056059:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.056061:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.056065:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.056068:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118720, 118720] 00000080:00200000:3.0:1788201707.056079:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.056082:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.056112:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.056116:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.056130:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486281216, count: 4096 00000080:00200000:3.0:1788201707.056139:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486281216, bytes: 4096 00000080:00200000:3.0:1788201707.056141:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.056145:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.056146:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486281216, 486285312) 0 0 00000080:00200000:3.0:1788201707.056148:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.056153:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118721, 118721] 00020000:00200000:3.0:1788201707.056156:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486281216 chunk: [486281216, 486285312) 486285312, 4096 00020000:00200000:3.0:1788201707.056158:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.056170:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118721, 118721] 00000008:00010000:3.0:1788201707.056174:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.056177:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.056184:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056192:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486281216 486285311) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056201:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056209:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056214:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.056220:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486281216, 486285312) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.056224:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118721 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.056228:0:17463:0:(rw26.c:898:ll_write_end()) pos 486281216, len 4096, copied 4096 00000080:00200000:3.0:1788201707.056231:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000931fc2e[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118721] 00000080:00200000:3.0:1788201707.056233:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003a0468fe fffffc0001029 2:0 ffff93c0c13b90e0 118721 lru 00000080:00200000:3.0:1788201707.056237:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d6f36b98 118721: 1< 2 - - > 2< 486281216 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70306 - -> 00000080:00200000:3.0:1788201707.056238:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000931fc2e 00000080:00200000:3.0:1788201707.056239:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.056240:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.056246:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.056254:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.056256:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.056257:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.056265:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.056267:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.056269:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.056272:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056277:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.056283:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.056284:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.056286:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.056288:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118721, 118721] 00000080:00200000:3.0:1788201707.056296:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.056298:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.056317:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.056320:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.056330:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486285312, count: 4096 00000080:00200000:3.0:1788201707.056336:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486285312, bytes: 4096 00000080:00200000:3.0:1788201707.056337:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.056340:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.056341:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486285312, 486289408) 0 0 00000080:00200000:3.0:1788201707.056342:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.056346:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118722, 118722] 00020000:00200000:3.0:1788201707.056348:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486285312 chunk: [486285312, 486289408) 486289408, 4096 00020000:00200000:3.0:1788201707.056349:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.056356:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118722, 118722] 00000008:00010000:3.0:1788201707.056359:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.056361:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.056366:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056371:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486285312 486289407) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056377:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056381:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056385:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.056389:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486285312, 486289408) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.056393:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118722 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.056396:0:17463:0:(rw26.c:898:ll_write_end()) pos 486285312, len 4096, copied 4096 00000080:00200000:3.0:1788201707.056399:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c31d258d[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118722] 00000080:00200000:3.0:1788201707.056400:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c5bb94f3 fffffc0001029 2:0 ffff93c0c13b9700 118722 lru 00000080:00200000:3.0:1788201707.056404:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009d279244 118722: 1< 2 - - > 2< 486285312 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70307 - -> 00000080:00200000:3.0:1788201707.056406:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c31d258d 00000080:00200000:3.0:1788201707.056406:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.056408:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.056413:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.056420:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.056422:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.056423:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.056429:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.056431:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.056432:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.056434:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056439:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.056444:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.056445:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.056447:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.056449:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118722, 118722] 00000080:00200000:3.0:1788201707.056456:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.056458:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.056476:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.056478:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.056488:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486289408, count: 4096 00000080:00200000:3.0:1788201707.056494:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486289408, bytes: 4096 00000080:00200000:3.0:1788201707.056496:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.056498:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.056499:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486289408, 486293504) 0 0 00000080:00200000:3.0:1788201707.056501:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.056504:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118723, 118723] 00020000:00200000:3.0:1788201707.056505:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486289408 chunk: [486289408, 486293504) 486293504, 4096 00020000:00200000:3.0:1788201707.056507:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.056513:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118723, 118723] 00000008:00010000:3.0:1788201707.056516:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.056518:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.056523:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056528:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486289408 486293503) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056534:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056538:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056542:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.056550:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486289408, 486293504) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.056553:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118723 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.056557:0:17463:0:(rw26.c:898:ll_write_end()) pos 486289408, len 4096, copied 4096 00000080:00200000:3.0:1788201707.056559:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000023c71aa2[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118723] 00000080:00200000:3.0:1788201707.056561:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004db84747 fffffc0001029 2:0 ffff93c0c13b9000 118723 lru 00000080:00200000:3.0:1788201707.056565:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001d8e9912 118723: 1< 2 - - > 2< 486289408 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70308 - -> 00000080:00200000:3.0:1788201707.056566:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000023c71aa2 00000080:00200000:3.0:1788201707.056567:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.056568:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.056573:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.056581:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.056582:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.056584:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.056589:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.056591:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.056592:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.056595:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056599:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.056605:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.056606:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.056609:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.056610:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118723, 118723] 00000080:00200000:3.0:1788201707.056618:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.056619:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.056637:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.056639:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.056649:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486293504, count: 4096 00000080:00200000:3.0:1788201707.056655:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486293504, bytes: 4096 00000080:00200000:3.0:1788201707.056657:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.056659:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.056660:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486293504, 486297600) 0 0 00000080:00200000:3.0:1788201707.056662:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.056665:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118724, 118724] 00020000:00200000:3.0:1788201707.056666:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486293504 chunk: [486293504, 486297600) 486297600, 4096 00020000:00200000:3.0:1788201707.056668:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.056674:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118724, 118724] 00000008:00010000:3.0:1788201707.056677:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.056679:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.056683:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056689:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486293504 486297599) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056694:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056699:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056706:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.056710:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486293504, 486297600) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.056713:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118724 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.056716:0:17463:0:(rw26.c:898:ll_write_end()) pos 486293504, len 4096, copied 4096 00000080:00200000:3.0:1788201707.056719:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001a66b60a[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118724] 00000080:00200000:3.0:1788201707.056720:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a09ab419 fffffc0001029 2:0 ffff93c0c13b9a80 118724 lru 00000080:00200000:3.0:1788201707.056724:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000069057a49 118724: 1< 2 - - > 2< 486293504 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70309 - -> 00000080:00200000:3.0:1788201707.056725:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001a66b60a 00000080:00200000:3.0:1788201707.056726:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.056727:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.056732:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.056740:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.056741:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.056743:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.056748:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.056750:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.056751:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.056754:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056758:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.056767:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.056768:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.056771:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.056772:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118724, 118724] 00000080:00200000:3.0:1788201707.056780:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.056781:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.056799:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.056801:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.056811:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486297600, count: 4096 00000080:00200000:3.0:1788201707.056817:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486297600, bytes: 4096 00000080:00200000:3.0:1788201707.056819:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.056821:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.056822:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486297600, 486301696) 0 0 00000080:00200000:3.0:1788201707.056823:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.056827:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118725, 118725] 00020000:00200000:3.0:1788201707.056841:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486297600 chunk: [486297600, 486301696) 486301696, 4096 00020000:00200000:3.0:1788201707.056843:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.056851:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118725, 118725] 00000008:00010000:3.0:1788201707.056854:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.056856:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.056860:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056866:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486297600 486301695) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056872:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056881:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056885:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.056889:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486297600, 486301696) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.056892:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118725 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.056895:0:17463:0:(rw26.c:898:ll_write_end()) pos 486297600, len 4096, copied 4096 00000080:00200000:3.0:1788201707.056897:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000af7b84b2[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118725] 00000080:00200000:3.0:1788201707.056899:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008c25a5a6 fffffc0001029 2:0 ffff93c0c13b9380 118725 lru 00000080:00200000:3.0:1788201707.056902:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001f5b34b4 118725: 1< 2 - - > 2< 486297600 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70310 - -> 00000080:00200000:3.0:1788201707.056903:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000af7b84b2 00000080:00200000:3.0:1788201707.056904:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.056905:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.056910:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.056918:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.056919:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.056921:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.056927:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.056928:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.056930:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.056932:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.056940:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.056945:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.056946:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.056948:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.056950:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118725, 118725] 00000080:00200000:3.0:1788201707.056957:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.056959:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.056977:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.056979:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.056989:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486301696, count: 4096 00000080:00200000:3.0:1788201707.056995:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486301696, bytes: 4096 00000080:00200000:3.0:1788201707.056997:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.056999:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.057000:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486301696, 486305792) 0 0 00000080:00200000:3.0:1788201707.057002:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.057005:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118726, 118726] 00020000:00200000:3.0:1788201707.057006:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486301696 chunk: [486301696, 486305792) 486305792, 4096 00020000:00200000:3.0:1788201707.057020:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.057028:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118726, 118726] 00000008:00010000:3.0:1788201707.057031:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.057032:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.057037:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057043:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486301696 486305791) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057052:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057057:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057061:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.057065:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486301696, 486305792) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.057068:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118726 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.057072:0:17463:0:(rw26.c:898:ll_write_end()) pos 486301696, len 4096, copied 4096 00000080:00200000:3.0:1788201707.057074:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000026d3578b[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118726] 00000080:00200000:3.0:1788201707.057075:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a6784b95 fffffc0001029 2:0 ffff93c0c13b98c0 118726 lru 00000080:00200000:3.0:1788201707.057079:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f588c3c9 118726: 1< 2 - - > 2< 486301696 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70311 - -> 00000080:00200000:3.0:1788201707.057080:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000026d3578b 00000080:00200000:3.0:1788201707.057081:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.057082:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.057087:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.057095:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.057097:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.057098:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.057104:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.057106:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.057107:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.057110:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057115:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.057119:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.057121:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.057123:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.057125:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118726, 118726] 00000080:00200000:3.0:1788201707.057132:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.057135:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.057153:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.057155:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.057165:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486305792, count: 4096 00000080:00200000:3.0:1788201707.057171:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486305792, bytes: 4096 00000080:00200000:3.0:1788201707.057172:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.057175:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.057176:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486305792, 486309888) 0 0 00000080:00200000:3.0:1788201707.057177:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.057181:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118727, 118727] 00020000:00200000:3.0:1788201707.057182:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486305792 chunk: [486305792, 486309888) 486309888, 4096 00020000:00200000:3.0:1788201707.057184:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.057190:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118727, 118727] 00000008:00010000:3.0:1788201707.057193:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.057195:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.057200:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057209:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486305792 486309887) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057214:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057219:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057222:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.057227:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486305792, 486309888) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.057230:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118727 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.057234:0:17463:0:(rw26.c:898:ll_write_end()) pos 486305792, len 4096, copied 4096 00000080:00200000:3.0:1788201707.057236:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000175745b0[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118727] 00000080:00200000:3.0:1788201707.057238:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f52d3adb fffffc0001029 2:0 ffff93c0c13b9ee0 118727 lru 00000080:00200000:3.0:1788201707.057242:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009722c3eb 118727: 1< 2 - - > 2< 486305792 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70312 - -> 00000080:00200000:3.0:1788201707.057243:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000175745b0 00000080:00200000:3.0:1788201707.057244:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.057245:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.057250:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.057258:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.057260:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.057261:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.057267:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.057268:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.057270:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.057272:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057277:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.057281:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.057282:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.057285:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.057287:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118727, 118727] 00000080:00200000:3.0:1788201707.057294:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.057296:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.057313:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.057316:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.057326:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486309888, count: 4096 00000080:00200000:3.0:1788201707.057331:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486309888, bytes: 4096 00000080:00200000:3.0:1788201707.057333:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.057336:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.057337:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486309888, 486313984) 0 0 00000080:00200000:3.0:1788201707.057338:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.057341:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118728, 118728] 00020000:00200000:3.0:1788201707.057343:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486309888 chunk: [486309888, 486313984) 486313984, 4096 00020000:00200000:3.0:1788201707.057344:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.057351:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118728, 118728] 00000008:00010000:3.0:1788201707.057354:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.057355:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.057360:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057366:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486309888 486313983) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057372:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057376:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057380:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.057385:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486309888, 486313984) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.057388:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118728 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.057391:0:17463:0:(rw26.c:898:ll_write_end()) pos 486309888, len 4096, copied 4096 00000080:00200000:3.0:1788201707.057394:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000027be3eaa[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118728] 00000080:00200000:3.0:1788201707.057395:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d1069f37 fffffc0001029 2:0 ffff93c0c13b97e0 118728 lru 00000080:00200000:3.0:1788201707.057399:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000002967905 118728: 1< 2 - - > 2< 486309888 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70313 - -> 00000080:00200000:3.0:1788201707.057400:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000027be3eaa 00000080:00200000:3.0:1788201707.057401:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.057402:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.057408:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.057415:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.057417:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.057418:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.057424:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.057425:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.057427:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.057429:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057434:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.057438:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.057440:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.057442:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.057444:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118728, 118728] 00000080:00200000:3.0:1788201707.057451:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.057453:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.057470:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.057472:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.057482:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486313984, count: 4096 00000080:00200000:3.0:1788201707.057488:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486313984, bytes: 4096 00000080:00200000:3.0:1788201707.057490:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.057492:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.057493:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486313984, 486318080) 0 0 00000080:00200000:3.0:1788201707.057495:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.057498:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118729, 118729] 00020000:00200000:3.0:1788201707.057500:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486313984 chunk: [486313984, 486318080) 486318080, 4096 00020000:00200000:3.0:1788201707.057501:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.057507:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118729, 118729] 00000008:00010000:3.0:1788201707.057511:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.057512:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.057517:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057522:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486313984 486318079) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057528:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057532:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057536:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.057540:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486313984, 486318080) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.057543:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118729 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.057546:0:17463:0:(rw26.c:898:ll_write_end()) pos 486313984, len 4096, copied 4096 00000080:00200000:3.0:1788201707.057548:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ae678ccb[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118729] 00000080:00200000:3.0:1788201707.057550:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000406950d5 fffffc0001029 2:0 ffff93c0c13b92a0 118729 lru 00000080:00200000:3.0:1788201707.057554:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007de2a18b 118729: 1< 2 - - > 2< 486313984 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70314 - -> 00000080:00200000:3.0:1788201707.057556:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ae678ccb 00000080:00200000:3.0:1788201707.057557:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.057558:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.057563:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.057573:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.057575:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.057576:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.057582:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.057583:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.057585:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.057587:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057592:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.057597:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.057598:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.057600:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.057602:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118729, 118729] 00000080:00200000:3.0:1788201707.057609:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.057611:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.057629:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.057631:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.057641:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486318080, count: 4096 00000080:00200000:3.0:1788201707.057647:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486318080, bytes: 4096 00000080:00200000:3.0:1788201707.057649:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.057651:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.057652:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486318080, 486322176) 0 0 00000080:00200000:3.0:1788201707.057654:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.057657:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118730, 118730] 00020000:00200000:3.0:1788201707.057659:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486318080 chunk: [486318080, 486322176) 486322176, 4096 00020000:00200000:3.0:1788201707.057660:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.057667:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118730, 118730] 00000008:00010000:3.0:1788201707.057670:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.057671:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.057676:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057681:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486318080 486322175) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057687:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057691:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057695:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.057700:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486318080, 486322176) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.057702:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118730 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.057706:0:17463:0:(rw26.c:898:ll_write_end()) pos 486318080, len 4096, copied 4096 00000080:00200000:3.0:1788201707.057708:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fb6b0904[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118730] 00000080:00200000:3.0:1788201707.057710:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000c3f78ec fffffc0001029 2:0 ffff93c0c13b9e00 118730 lru 00000080:00200000:3.0:1788201707.057714:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cb260ac2 118730: 1< 2 - - > 2< 486318080 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70315 - -> 00000080:00200000:3.0:1788201707.057715:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fb6b0904 00000080:00200000:3.0:1788201707.057716:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.057717:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.057722:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.057730:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.057731:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.057733:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.057739:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.057740:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.057742:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.057744:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057749:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.057753:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.057754:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.057756:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.057758:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118730, 118730] 00000080:00200000:3.0:1788201707.057766:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.057767:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.057785:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.057787:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.057797:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486322176, count: 4096 00000080:00200000:3.0:1788201707.057803:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486322176, bytes: 4096 00000080:00200000:3.0:1788201707.057804:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.057807:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.057808:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486322176, 486326272) 0 0 00000080:00200000:3.0:1788201707.057810:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.057813:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118731, 118731] 00020000:00200000:3.0:1788201707.057815:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486322176 chunk: [486322176, 486326272) 486326272, 4096 00020000:00200000:3.0:1788201707.057816:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.057823:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118731, 118731] 00000008:00010000:3.0:1788201707.057826:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.057827:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.057844:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057851:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486322176 486326271) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057856:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057860:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057865:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.057869:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486322176, 486326272) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.057873:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118731 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.057876:0:17463:0:(rw26.c:898:ll_write_end()) pos 486322176, len 4096, copied 4096 00000080:00200000:3.0:1788201707.057879:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000062be5538[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118731] 00000080:00200000:3.0:1788201707.057881:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007c5218a5 fffffc0001029 2:0 ffff93c0c13b91c0 118731 lru 00000080:00200000:3.0:1788201707.057884:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000036b6dea1 118731: 1< 2 - - > 2< 486322176 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70316 - -> 00000080:00200000:3.0:1788201707.057887:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000062be5538 00000080:00200000:3.0:1788201707.057887:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.057889:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.057894:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.057902:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.057904:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.057905:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.057911:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.057912:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.057914:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.057916:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.057921:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.057926:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.057927:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.057929:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.057931:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118731, 118731] 00000080:00200000:3.0:1788201707.057938:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.057940:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.057958:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.057960:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.057970:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486326272, count: 4096 00000080:00200000:3.0:1788201707.057976:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486326272, bytes: 4096 00000080:00200000:3.0:1788201707.057978:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.057980:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.057981:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486326272, 486330368) 0 0 00000080:00200000:3.0:1788201707.057983:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.057986:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118732, 118732] 00020000:00200000:3.0:1788201707.057988:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486326272 chunk: [486326272, 486330368) 486330368, 4096 00020000:00200000:3.0:1788201707.057989:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.057996:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118732, 118732] 00000008:00010000:3.0:1788201707.057999:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.058000:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.058005:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058024:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486326272 486330367) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058029:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058034:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058038:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.058042:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486326272, 486330368) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.058046:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118732 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.058049:0:17463:0:(rw26.c:898:ll_write_end()) pos 486326272, len 4096, copied 4096 00000080:00200000:3.0:1788201707.058052:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000742bb163[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118732] 00000080:00200000:3.0:1788201707.058054:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e4548662 fffffc0001029 2:0 ffff93c0c13b9620 118732 lru 00000080:00200000:3.0:1788201707.058057:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000055c09cd3 118732: 1< 2 - - > 2< 486326272 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70317 - -> 00000080:00200000:3.0:1788201707.058059:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000742bb163 00000080:00200000:3.0:1788201707.058060:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.058061:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.058066:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.058074:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.058075:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.058077:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.058083:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.058084:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.058086:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.058088:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058093:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.058098:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.058099:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.058101:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.058103:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118732, 118732] 00000080:00200000:3.0:1788201707.058111:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.058112:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.058130:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.058132:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.058143:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486330368, count: 4096 00000080:00200000:3.0:1788201707.058148:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486330368, bytes: 4096 00000080:00200000:3.0:1788201707.058150:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.058153:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.058154:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486330368, 486334464) 0 0 00000080:00200000:3.0:1788201707.058155:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.058158:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118733, 118733] 00020000:00200000:3.0:1788201707.058160:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486330368 chunk: [486330368, 486334464) 486334464, 4096 00020000:00200000:3.0:1788201707.058162:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.058168:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118733, 118733] 00000008:00010000:3.0:1788201707.058171:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.058173:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.058178:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058183:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486330368 486334463) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058188:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058193:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058197:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.058201:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486330368, 486334464) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.058204:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118733 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.058208:0:17463:0:(rw26.c:898:ll_write_end()) pos 486330368, len 4096, copied 4096 00000080:00200000:3.0:1788201707.058210:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004bf63984[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118733] 00000080:00200000:3.0:1788201707.058212:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007aa6a8a9 fffffc0001029 2:0 ffff93c0c13b9d20 118733 lru 00000080:00200000:3.0:1788201707.058216:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000dc916638 118733: 1< 2 - - > 2< 486330368 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70318 - -> 00000080:00200000:3.0:1788201707.058218:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004bf63984 00000080:00200000:3.0:1788201707.058218:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.058220:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.058225:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.058232:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.058234:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.058235:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.058241:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.058243:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.058245:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.058247:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058252:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.058257:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.058258:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.058260:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.058262:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118733, 118733] 00000080:00200000:3.0:1788201707.058269:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.058271:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.058289:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.058291:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.058301:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486334464, count: 4096 00000080:00200000:3.0:1788201707.058308:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486334464, bytes: 4096 00000080:00200000:3.0:1788201707.058309:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.058311:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.058312:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486334464, 486338560) 0 0 00000080:00200000:3.0:1788201707.058314:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.058317:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118734, 118734] 00020000:00200000:3.0:1788201707.058319:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486334464 chunk: [486334464, 486338560) 486338560, 4096 00020000:00200000:3.0:1788201707.058320:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.058327:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118734, 118734] 00000008:00010000:3.0:1788201707.058330:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.058331:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.058336:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058342:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486334464 486338559) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058347:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058352:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058356:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.058360:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486334464, 486338560) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.058364:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118734 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.058367:0:17463:0:(rw26.c:898:ll_write_end()) pos 486334464, len 4096, copied 4096 00000080:00200000:3.0:1788201707.058369:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b1b82e4f[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118734] 00000080:00200000:3.0:1788201707.058371:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000c5768f3 fffffc0001029 2:0 ffff93c0c13b99a0 118734 lru 00000080:00200000:3.0:1788201707.058375:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a40fd71f 118734: 1< 2 - - > 2< 486334464 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70319 - -> 00000080:00200000:3.0:1788201707.058376:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b1b82e4f 00000080:00200000:3.0:1788201707.058377:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.058378:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.058383:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.058390:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.058392:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.058393:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.058399:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.058401:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.058402:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.058404:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058409:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.058414:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.058415:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.058417:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.058419:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118734, 118734] 00000080:00200000:3.0:1788201707.058426:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.058428:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.058446:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.058448:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.058458:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486338560, count: 4096 00000080:00200000:3.0:1788201707.058464:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486338560, bytes: 4096 00000080:00200000:3.0:1788201707.058465:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.058468:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.058469:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486338560, 486342656) 0 0 00000080:00200000:3.0:1788201707.058470:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.058473:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118735, 118735] 00020000:00200000:3.0:1788201707.058475:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486338560 chunk: [486338560, 486342656) 486342656, 4096 00020000:00200000:3.0:1788201707.058477:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.058483:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118735, 118735] 00000008:00010000:3.0:1788201707.058486:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.058487:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.058492:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058498:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486338560 486342655) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058503:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058507:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058511:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.058519:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486338560, 486342656) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.058522:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118735 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.058525:0:17463:0:(rw26.c:898:ll_write_end()) pos 486338560, len 4096, copied 4096 00000080:00200000:3.0:1788201707.058528:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003b6ffb89[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118735] 00000080:00200000:3.0:1788201707.058530:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ab25d30f fffffc0001029 2:0 ffff93c0c13bad20 118735 lru 00000080:00200000:3.0:1788201707.058533:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000367a94c7 118735: 1< 2 - - > 2< 486338560 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70320 - -> 00000080:00200000:3.0:1788201707.058535:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003b6ffb89 00000080:00200000:3.0:1788201707.058536:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.058537:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.058542:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.058549:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.058551:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.058552:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.058558:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.058559:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.058561:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.058563:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058568:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.058573:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.058574:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.058577:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.058579:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118735, 118735] 00000080:00200000:3.0:1788201707.058586:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.058588:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.058605:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.058608:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.058618:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486342656, count: 4096 00000080:00200000:3.0:1788201707.058624:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486342656, bytes: 4096 00000080:00200000:3.0:1788201707.058625:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.058628:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.058629:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486342656, 486346752) 0 0 00000080:00200000:3.0:1788201707.058630:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.058633:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118736, 118736] 00020000:00200000:3.0:1788201707.058635:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486342656 chunk: [486342656, 486346752) 486346752, 4096 00020000:00200000:3.0:1788201707.058637:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.058643:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118736, 118736] 00000008:00010000:3.0:1788201707.058646:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.058648:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.058652:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058658:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486342656 486346751) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058663:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058668:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058675:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.058679:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486342656, 486346752) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.058682:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118736 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.058685:0:17463:0:(rw26.c:898:ll_write_end()) pos 486342656, len 4096, copied 4096 00000080:00200000:3.0:1788201707.058687:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a1cfa32f[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118736] 00000080:00200000:3.0:1788201707.058689:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a89f5bbd fffffc0001029 2:0 ffff93c0c13ba9a0 118736 lru 00000080:00200000:3.0:1788201707.058693:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000060c920b0 118736: 1< 2 - - > 2< 486342656 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70321 - -> 00000080:00200000:3.0:1788201707.058694:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a1cfa32f 00000080:00200000:3.0:1788201707.058694:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.058696:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.058701:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.058708:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.058710:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.058711:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.058717:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.058718:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.058720:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.058722:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058727:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.058735:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.058737:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.058739:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.058741:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118736, 118736] 00000080:00200000:3.0:1788201707.058748:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.058750:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.058768:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.058770:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.058780:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486346752, count: 4096 00000080:00200000:3.0:1788201707.058785:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486346752, bytes: 4096 00000080:00200000:3.0:1788201707.058787:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.058789:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.058790:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486346752, 486350848) 0 0 00000080:00200000:3.0:1788201707.058792:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.058795:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118737, 118737] 00020000:00200000:3.0:1788201707.058797:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486346752 chunk: [486346752, 486350848) 486350848, 4096 00020000:00200000:3.0:1788201707.058798:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.058805:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118737, 118737] 00000008:00010000:3.0:1788201707.058808:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.058809:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.058814:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058819:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486346752 486350847) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058825:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058846:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058851:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.058855:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486346752, 486350848) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.058858:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118737 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.058862:0:17463:0:(rw26.c:898:ll_write_end()) pos 486346752, len 4096, copied 4096 00000080:00200000:3.0:1788201707.058864:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000056eededc[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118737] 00000080:00200000:3.0:1788201707.058866:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000eb8dc66 fffffc0001029 2:0 ffff93c0c13bab60 118737 lru 00000080:00200000:3.0:1788201707.058869:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001bc9c0f1 118737: 1< 2 - - > 2< 486346752 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70322 - -> 00000080:00200000:3.0:1788201707.058871:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000056eededc 00000080:00200000:3.0:1788201707.058871:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.058873:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.058878:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.058886:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.058887:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.058889:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.058895:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.058896:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.058898:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.058900:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058909:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.058913:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.058915:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.058917:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.058919:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118737, 118737] 00000080:00200000:3.0:1788201707.058926:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.058928:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.058946:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.058948:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.058958:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486350848, count: 4096 00000080:00200000:3.0:1788201707.058964:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486350848, bytes: 4096 00000080:00200000:3.0:1788201707.058965:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.058967:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.058968:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486350848, 486354944) 0 0 00000080:00200000:3.0:1788201707.058970:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.058973:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118738, 118738] 00020000:00200000:3.0:1788201707.058975:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486350848 chunk: [486350848, 486354944) 486354944, 4096 00020000:00200000:3.0:1788201707.058976:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.058983:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118738, 118738] 00000008:00010000:3.0:1788201707.058986:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.058987:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.058992:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.058998:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486350848 486354943) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059018:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059023:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059028:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.059032:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486350848, 486354944) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.059035:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118738 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.059039:0:17463:0:(rw26.c:898:ll_write_end()) pos 486350848, len 4096, copied 4096 00000080:00200000:3.0:1788201707.059041:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a1d81fc4[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118738] 00000080:00200000:3.0:1788201707.059043:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fab3a3d0 fffffc0001029 2:0 ffff93c0c13bac40 118738 lru 00000080:00200000:3.0:1788201707.059046:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f037eba0 118738: 1< 2 - - > 2< 486350848 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70323 - -> 00000080:00200000:3.0:1788201707.059048:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a1d81fc4 00000080:00200000:3.0:1788201707.059049:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.059050:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.059055:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.059063:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.059064:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.059065:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.059071:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.059073:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.059075:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.059077:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059082:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.059087:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.059089:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.059091:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.059093:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118738, 118738] 00000080:00200000:3.0:1788201707.059100:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.059102:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.059120:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.059122:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.059132:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486354944, count: 4096 00000080:00200000:3.0:1788201707.059138:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486354944, bytes: 4096 00000080:00200000:3.0:1788201707.059140:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.059142:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.059143:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486354944, 486359040) 0 0 00000080:00200000:3.0:1788201707.059144:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.059148:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118739, 118739] 00020000:00200000:3.0:1788201707.059149:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486354944 chunk: [486354944, 486359040) 486359040, 4096 00020000:00200000:3.0:1788201707.059151:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.059157:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118739, 118739] 00000008:00010000:3.0:1788201707.059160:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.059162:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.059167:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059176:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486354944 486359039) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059181:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059185:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059189:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.059194:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486354944, 486359040) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.059196:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118739 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.059200:0:17463:0:(rw26.c:898:ll_write_end()) pos 486354944, len 4096, copied 4096 00000080:00200000:3.0:1788201707.059202:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005eb34dce[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118739] 00000080:00200000:3.0:1788201707.059204:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b8c7a971 fffffc0001029 2:0 ffff93c0c13ba540 118739 lru 00000080:00200000:3.0:1788201707.059208:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000217af31d 118739: 1< 2 - - > 2< 486354944 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70324 - -> 00000080:00200000:3.0:1788201707.059209:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005eb34dce 00000080:00200000:3.0:1788201707.059210:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.059211:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.059216:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.059223:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.059225:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.059226:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.059232:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.059234:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.059236:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.059238:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059243:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.059247:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.059249:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.059251:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.059253:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118739, 118739] 00000080:00200000:3.0:1788201707.059260:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.059262:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.059279:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.059282:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.059291:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486359040, count: 4096 00000080:00200000:3.0:1788201707.059297:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486359040, bytes: 4096 00000080:00200000:3.0:1788201707.059299:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.059301:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.059302:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486359040, 486363136) 0 0 00000080:00200000:3.0:1788201707.059304:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.059307:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118740, 118740] 00020000:00200000:3.0:1788201707.059309:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486359040 chunk: [486359040, 486363136) 486363136, 4096 00020000:00200000:3.0:1788201707.059310:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.059317:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118740, 118740] 00000008:00010000:3.0:1788201707.059320:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.059321:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.059326:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059331:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486359040 486363135) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059337:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059341:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059345:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.059350:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486359040, 486363136) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.059353:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118740 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.059356:0:17463:0:(rw26.c:898:ll_write_end()) pos 486359040, len 4096, copied 4096 00000080:00200000:3.0:1788201707.059359:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000762f5dfc[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118740] 00000080:00200000:3.0:1788201707.059360:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005594b3ce fffffc0001029 2:0 ffff93c0c13ba460 118740 lru 00000080:00200000:3.0:1788201707.059364:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003623f11b 118740: 1< 2 - - > 2< 486359040 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70325 - -> 00000080:00200000:3.0:1788201707.059365:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000762f5dfc 00000080:00200000:3.0:1788201707.059366:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.059368:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.059373:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.059380:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.059382:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.059383:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.059389:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.059390:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.059392:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.059394:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059399:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.059403:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.059405:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.059407:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.059409:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118740, 118740] 00000080:00200000:3.0:1788201707.059416:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.059418:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.059435:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.059438:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.059448:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486363136, count: 4096 00000080:00200000:3.0:1788201707.059454:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486363136, bytes: 4096 00000080:00200000:3.0:1788201707.059456:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.059458:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.059459:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486363136, 486367232) 0 0 00000080:00200000:3.0:1788201707.059460:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.059463:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118741, 118741] 00020000:00200000:3.0:1788201707.059465:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486363136 chunk: [486363136, 486367232) 486367232, 4096 00020000:00200000:3.0:1788201707.059466:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.059473:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118741, 118741] 00000008:00010000:3.0:1788201707.059476:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.059478:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.059482:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059488:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486363136 486367231) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059493:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059497:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059501:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.059506:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486363136, 486367232) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.059509:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118741 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.059512:0:17463:0:(rw26.c:898:ll_write_end()) pos 486363136, len 4096, copied 4096 00000080:00200000:3.0:1788201707.059514:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ee08de02[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118741] 00000080:00200000:3.0:1788201707.059516:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b9de89b0 fffffc0001029 2:0 ffff93c0c13ba0e0 118741 lru 00000080:00200000:3.0:1788201707.059520:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000dd7064c6 118741: 1< 2 - - > 2< 486363136 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70326 - -> 00000080:00200000:3.0:1788201707.059521:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ee08de02 00000080:00200000:3.0:1788201707.059522:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.059523:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.059528:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.059539:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.059541:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.059542:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.059547:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.059549:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.059551:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.059553:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059558:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.059562:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.059563:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.059566:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.059567:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118741, 118741] 00000080:00200000:3.0:1788201707.059575:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.059577:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.059595:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.059597:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.059606:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486367232, count: 4096 00000080:00200000:3.0:1788201707.059612:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486367232, bytes: 4096 00000080:00200000:3.0:1788201707.059614:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.059616:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.059617:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486367232, 486371328) 0 0 00000080:00200000:3.0:1788201707.059619:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.059622:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118742, 118742] 00020000:00200000:3.0:1788201707.059624:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486367232 chunk: [486367232, 486371328) 486371328, 4096 00020000:00200000:3.0:1788201707.059625:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.059632:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118742, 118742] 00000008:00010000:3.0:1788201707.059635:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.059636:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.059641:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059647:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486367232 486371327) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059652:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059656:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059660:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.059665:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486367232, 486371328) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.059668:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118742 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.059671:0:17463:0:(rw26.c:898:ll_write_end()) pos 486367232, len 4096, copied 4096 00000080:00200000:3.0:1788201707.059673:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a2de8e7a[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118742] 00000080:00200000:3.0:1788201707.059675:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fce9e8e9 fffffc0001029 2:0 ffff93c0c13ba700 118742 lru 00000080:00200000:3.0:1788201707.059679:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002328c6b1 118742: 1< 2 - - > 2< 486367232 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70327 - -> 00000080:00200000:3.0:1788201707.059680:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a2de8e7a 00000080:00200000:3.0:1788201707.059681:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.059683:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.059688:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.059695:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.059697:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.059698:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.059704:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.059705:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.059707:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.059709:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059714:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.059719:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.059720:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.059722:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.059724:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118742, 118742] 00000080:00200000:3.0:1788201707.059731:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.059733:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.059750:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.059753:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.059762:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486371328, count: 4096 00000080:00200000:3.0:1788201707.059768:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486371328, bytes: 4096 00000080:00200000:3.0:1788201707.059770:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.059772:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.059773:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486371328, 486375424) 0 0 00000080:00200000:3.0:1788201707.059775:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.059778:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118743, 118743] 00020000:00200000:3.0:1788201707.059780:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486371328 chunk: [486371328, 486375424) 486375424, 4096 00020000:00200000:3.0:1788201707.059781:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.059788:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118743, 118743] 00000008:00010000:3.0:1788201707.059791:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.059792:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.059797:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059803:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486371328 486375423) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059808:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059812:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059816:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.059821:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486371328, 486375424) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.059824:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118743 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.059827:0:17463:0:(rw26.c:898:ll_write_end()) pos 486371328, len 4096, copied 4096 00000080:00200000:3.0:1788201707.059842:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c4f5dc26[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118743] 00000080:00200000:3.0:1788201707.059844:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008ec56cda fffffc0001029 2:0 ffff93c0c13ba000 118743 lru 00000080:00200000:3.0:1788201707.059848:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000011c25316 118743: 1< 2 - - > 2< 486371328 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70328 - -> 00000080:00200000:3.0:1788201707.059850:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c4f5dc26 00000080:00200000:3.0:1788201707.059851:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.059853:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.059858:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.059866:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.059868:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.059869:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.059875:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.059877:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.059878:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.059881:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059885:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.059890:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.059892:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.059894:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.059896:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118743, 118743] 00000080:00200000:3.0:1788201707.059903:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.059905:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.059923:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.059925:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.059935:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486375424, count: 4096 00000080:00200000:3.0:1788201707.059941:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486375424, bytes: 4096 00000080:00200000:3.0:1788201707.059943:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.059945:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.059946:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486375424, 486379520) 0 0 00000080:00200000:3.0:1788201707.059948:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.059951:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118744, 118744] 00020000:00200000:3.0:1788201707.059953:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486375424 chunk: [486375424, 486379520) 486379520, 4096 00020000:00200000:3.0:1788201707.059954:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.059960:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118744, 118744] 00000008:00010000:3.0:1788201707.059963:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.059965:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.059969:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059975:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486375424 486379519) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059981:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059985:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.059989:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.059993:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486375424, 486379520) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.059996:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118744 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.060000:0:17463:0:(rw26.c:898:ll_write_end()) pos 486375424, len 4096, copied 4096 00000080:00200000:3.0:1788201707.060002:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e71d8cd4[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118744] 00000080:00200000:3.0:1788201707.060004:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d5a819dc fffffc0001029 2:0 ffff93c0c13baa80 118744 lru 00000080:00200000:3.0:1788201707.060017:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000faf46297 118744: 1< 2 - - > 2< 486375424 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70329 - -> 00000080:00200000:3.0:1788201707.060019:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e71d8cd4 00000080:00200000:3.0:1788201707.060020:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.060021:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.060027:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.060035:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.060037:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.060038:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.060044:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.060046:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.060047:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.060050:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060055:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.060060:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.060061:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.060063:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.060065:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118744, 118744] 00000080:00200000:3.0:1788201707.060072:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.060074:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.060092:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.060094:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.060105:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486379520, count: 4096 00000080:00200000:3.0:1788201707.060111:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486379520, bytes: 4096 00000080:00200000:3.0:1788201707.060113:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.060115:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.060116:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486379520, 486383616) 0 0 00000080:00200000:3.0:1788201707.060117:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.060120:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118745, 118745] 00020000:00200000:3.0:1788201707.060122:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486379520 chunk: [486379520, 486383616) 486383616, 4096 00020000:00200000:3.0:1788201707.060124:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.060130:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118745, 118745] 00000008:00010000:3.0:1788201707.060133:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.060135:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.060139:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060145:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486379520 486383615) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060151:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060156:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060160:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.060164:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486379520, 486383616) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.060167:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118745 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.060171:0:17463:0:(rw26.c:898:ll_write_end()) pos 486379520, len 4096, copied 4096 00000080:00200000:3.0:1788201707.060173:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e6877297[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118745] 00000080:00200000:3.0:1788201707.060175:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f155e26c fffffc0001029 2:0 ffff93c0c13ba380 118745 lru 00000080:00200000:3.0:1788201707.060179:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000527e4e58 118745: 1< 2 - - > 2< 486379520 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70330 - -> 00000080:00200000:3.0:1788201707.060180:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e6877297 00000080:00200000:3.0:1788201707.060181:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.060182:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.060187:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.060195:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.060197:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.060198:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.060204:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.060206:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.060207:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.060210:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060214:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.060219:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.060220:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.060222:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.060224:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118745, 118745] 00000080:00200000:3.0:1788201707.060231:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.060233:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.060251:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.060253:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.060263:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486383616, count: 4096 00000080:00200000:3.0:1788201707.060269:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486383616, bytes: 4096 00000080:00200000:3.0:1788201707.060271:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.060273:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.060274:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486383616, 486387712) 0 0 00000080:00200000:3.0:1788201707.060275:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.060278:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118746, 118746] 00020000:00200000:3.0:1788201707.060280:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486383616 chunk: [486383616, 486387712) 486387712, 4096 00020000:00200000:3.0:1788201707.060282:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.060288:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118746, 118746] 00000008:00010000:3.0:1788201707.060291:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.060293:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.060297:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060303:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486383616 486387711) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060308:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060313:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060317:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.060320:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486383616, 486387712) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.060324:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118746 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.060327:0:17463:0:(rw26.c:898:ll_write_end()) pos 486383616, len 4096, copied 4096 00000080:00200000:3.0:1788201707.060330:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fab4aebd[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118746] 00000080:00200000:3.0:1788201707.060331:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000def44777 fffffc0001029 2:0 ffff93c0c13ba8c0 118746 lru 00000080:00200000:3.0:1788201707.060335:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f9f4b725 118746: 1< 2 - - > 2< 486383616 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70331 - -> 00000080:00200000:3.0:1788201707.060337:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fab4aebd 00000080:00200000:3.0:1788201707.060337:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.060339:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.060344:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.060351:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.060353:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.060354:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.060360:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.060361:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.060363:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.060365:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060370:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.060375:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.060376:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.060378:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.060380:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118746, 118746] 00000080:00200000:3.0:1788201707.060387:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.060389:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.060407:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.060409:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.060419:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486387712, count: 4096 00000080:00200000:3.0:1788201707.060425:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486387712, bytes: 4096 00000080:00200000:3.0:1788201707.060427:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.060429:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.060430:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486387712, 486391808) 0 0 00000080:00200000:3.0:1788201707.060431:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.060434:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118747, 118747] 00020000:00200000:3.0:1788201707.060436:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486387712 chunk: [486387712, 486391808) 486391808, 4096 00020000:00200000:3.0:1788201707.060438:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.060444:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118747, 118747] 00000008:00010000:3.0:1788201707.060447:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.060449:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.060453:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060459:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486387712 486391807) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060464:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060469:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060473:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.060480:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486387712, 486391808) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.060483:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118747 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.060487:0:17463:0:(rw26.c:898:ll_write_end()) pos 486387712, len 4096, copied 4096 00000080:00200000:3.0:1788201707.060489:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005368c94b[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118747] 00000080:00200000:3.0:1788201707.060491:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b1e9d732 fffffc0001029 2:0 ffff93c0c13baee0 118747 lru 00000080:00200000:3.0:1788201707.060494:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005d36731b 118747: 1< 2 - - > 2< 486387712 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70332 - -> 00000080:00200000:3.0:1788201707.060496:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005368c94b 00000080:00200000:3.0:1788201707.060497:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.060498:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.060503:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.060510:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.060512:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.060513:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.060519:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.060520:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.060522:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.060524:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060529:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.060533:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.060535:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.060537:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.060539:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118747, 118747] 00000080:00200000:3.0:1788201707.060546:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.060548:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.060566:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.060568:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.060578:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486391808, count: 4096 00000080:00200000:3.0:1788201707.060584:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486391808, bytes: 4096 00000080:00200000:3.0:1788201707.060585:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.060588:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.060589:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486391808, 486395904) 0 0 00000080:00200000:3.0:1788201707.060590:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.060593:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118748, 118748] 00020000:00200000:3.0:1788201707.060595:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486391808 chunk: [486391808, 486395904) 486395904, 4096 00020000:00200000:3.0:1788201707.060597:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.060603:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118748, 118748] 00000008:00010000:3.0:1788201707.060606:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.060608:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.060612:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060618:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486391808 486395903) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060623:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060628:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060635:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.060639:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486391808, 486395904) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.060642:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118748 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.060646:0:17463:0:(rw26.c:898:ll_write_end()) pos 486391808, len 4096, copied 4096 00000080:00200000:3.0:1788201707.060648:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e2e89a23[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118748] 00000080:00200000:3.0:1788201707.060650:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000560e29b0 fffffc0001029 2:0 ffff93c0c13ba7e0 118748 lru 00000080:00200000:3.0:1788201707.060653:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007cab9a97 118748: 1< 2 - - > 2< 486391808 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70333 - -> 00000080:00200000:3.0:1788201707.060655:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e2e89a23 00000080:00200000:3.0:1788201707.060655:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.060656:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.060662:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.060669:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.060671:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.060672:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.060677:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.060679:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.060680:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.060683:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060688:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.060696:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.060697:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.060699:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.060701:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118748, 118748] 00000080:00200000:3.0:1788201707.060708:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.060710:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.060728:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.060730:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.060740:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486395904, count: 4096 00000080:00200000:3.0:1788201707.060746:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486395904, bytes: 4096 00000080:00200000:3.0:1788201707.060747:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.060749:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.060750:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486395904, 486400000) 0 0 00000080:00200000:3.0:1788201707.060752:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.060755:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118749, 118749] 00020000:00200000:3.0:1788201707.060757:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486395904 chunk: [486395904, 486400000) 486400000, 4096 00020000:00200000:3.0:1788201707.060759:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.060765:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118749, 118749] 00000008:00010000:3.0:1788201707.060768:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.060769:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.060774:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060780:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486395904 486399999) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060785:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060793:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060797:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.060801:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486395904, 486400000) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.060804:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118749 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.060807:0:17463:0:(rw26.c:898:ll_write_end()) pos 486395904, len 4096, copied 4096 00000080:00200000:3.0:1788201707.060809:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bbee97a5[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118749] 00000080:00200000:3.0:1788201707.060811:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b48c0b99 fffffc0001029 2:0 ffff93c0c13ba2a0 118749 lru 00000080:00200000:3.0:1788201707.060814:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000054bdf48a 118749: 1< 2 - - > 2< 486395904 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70334 - -> 00000080:00200000:3.0:1788201707.060815:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bbee97a5 00000080:00200000:3.0:1788201707.060816:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.060817:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.060822:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.060842:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.060844:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.060846:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.060851:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.060853:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.060855:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.060857:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060865:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.060870:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.060871:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.060874:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.060875:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118749, 118749] 00000080:00200000:3.0:1788201707.060883:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.060885:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.060903:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.060905:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.060915:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486400000, count: 4096 00000080:00200000:3.0:1788201707.060921:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486400000, bytes: 4096 00000080:00200000:3.0:1788201707.060922:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.060925:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.060926:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486400000, 486404096) 0 0 00000080:00200000:3.0:1788201707.060927:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.060930:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118750, 118750] 00020000:00200000:3.0:1788201707.060932:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486400000 chunk: [486400000, 486404096) 486404096, 4096 00020000:00200000:3.0:1788201707.060933:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.060940:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118750, 118750] 00000008:00010000:3.0:1788201707.060943:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.060945:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.060949:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060955:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486400000 486404095) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060964:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060968:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.060972:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.060977:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486400000, 486404096) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.060980:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118750 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.060983:0:17463:0:(rw26.c:898:ll_write_end()) pos 486400000, len 4096, copied 4096 00000080:00200000:3.0:1788201707.060986:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000059f69e47[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118750] 00000080:00200000:3.0:1788201707.060987:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cb0a0fe7 fffffc0001029 2:0 ffff93c0c13bae00 118750 lru 00000080:00200000:3.0:1788201707.060991:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000233639c3 118750: 1< 2 - - > 2< 486400000 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70335 - -> 00000080:00200000:3.0:1788201707.060993:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000059f69e47 00000080:00200000:3.0:1788201707.060994:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.060995:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.061000:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.061018:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.061020:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.061022:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.061028:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.061030:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.061031:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.061034:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061039:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.061044:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.061045:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.061048:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.061049:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118750, 118750] 00000080:00200000:3.0:1788201707.061057:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.061059:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.061076:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.061078:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.061088:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486404096, count: 4096 00000080:00200000:3.0:1788201707.061095:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486404096, bytes: 4096 00000080:00200000:3.0:1788201707.061096:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.061099:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.061100:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486404096, 486408192) 0 0 00000080:00200000:3.0:1788201707.061101:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.061105:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118751, 118751] 00020000:00200000:3.0:1788201707.061106:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486404096 chunk: [486404096, 486408192) 486408192, 4096 00020000:00200000:3.0:1788201707.061108:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.061114:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118751, 118751] 00000008:00010000:3.0:1788201707.061117:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.061119:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.061123:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061132:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486404096 486408191) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061138:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061142:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061146:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.061150:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486404096, 486408192) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.061153:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118751 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.061156:0:17463:0:(rw26.c:898:ll_write_end()) pos 486404096, len 4096, copied 4096 00000080:00200000:3.0:1788201707.061158:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000fe27c0c[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118751] 00000080:00200000:3.0:1788201707.061160:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000610f3551 fffffc0001029 2:0 ffff93c0c13ba1c0 118751 lru 00000080:00200000:3.0:1788201707.061163:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002780e5b2 118751: 1< 2 - - > 2< 486404096 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70336 - -> 00000080:00200000:3.0:1788201707.061165:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000fe27c0c 00000080:00200000:3.0:1788201707.061165:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.061166:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.061171:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.061179:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.061180:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.061182:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.061188:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.061189:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.061191:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.061193:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061197:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.061202:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.061203:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.061206:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.061207:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118751, 118751] 00000080:00200000:3.0:1788201707.061215:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.061217:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.061234:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.061237:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.061247:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486408192, count: 4096 00000080:00200000:3.0:1788201707.061253:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486408192, bytes: 4096 00000080:00200000:3.0:1788201707.061254:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.061257:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.061258:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486408192, 486412288) 0 0 00000080:00200000:3.0:1788201707.061259:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.061262:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118752, 118752] 00020000:00200000:3.0:1788201707.061264:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486408192 chunk: [486408192, 486412288) 486412288, 4096 00020000:00200000:3.0:1788201707.061266:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.061272:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118752, 118752] 00000008:00010000:3.0:1788201707.061275:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.061276:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.061281:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061287:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486408192 486412287) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061293:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061297:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061301:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.061305:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486408192, 486412288) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.061308:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118752 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.061312:0:17463:0:(rw26.c:898:ll_write_end()) pos 486408192, len 4096, copied 4096 00000080:00200000:3.0:1788201707.061314:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002718453e[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118752] 00000080:00200000:3.0:1788201707.061316:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ed129f79 fffffc0001029 2:0 ffff93c0c13ba620 118752 lru 00000080:00200000:3.0:1788201707.061319:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000093eea182 118752: 1< 2 - - > 2< 486408192 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70337 - -> 00000080:00200000:3.0:1788201707.061320:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002718453e 00000080:00200000:3.0:1788201707.061321:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.061323:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.061328:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.061335:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.061337:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.061338:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.061344:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.061346:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.061347:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.061350:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061354:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.061359:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.061360:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.061362:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.061364:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118752, 118752] 00000080:00200000:3.0:1788201707.061372:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.061374:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.061391:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.061393:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.061403:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486412288, count: 4096 00000080:00200000:3.0:1788201707.061409:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486412288, bytes: 4096 00000080:00200000:3.0:1788201707.061411:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.061413:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.061414:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486412288, 486416384) 0 0 00000080:00200000:3.0:1788201707.061416:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.061419:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118753, 118753] 00020000:00200000:3.0:1788201707.061421:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486412288 chunk: [486412288, 486416384) 486416384, 4096 00020000:00200000:3.0:1788201707.061422:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.061429:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118753, 118753] 00000008:00010000:3.0:1788201707.061432:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.061433:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.061438:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061444:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486412288 486416383) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061449:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061454:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061457:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.061462:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486412288, 486416384) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.061465:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118753 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.061469:0:17463:0:(rw26.c:898:ll_write_end()) pos 486412288, len 4096, copied 4096 00000080:00200000:3.0:1788201707.061471:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000034bce8e[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118753] 00000080:00200000:3.0:1788201707.061473:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001c0e416c fffffc0001029 2:0 ffff93c0c13bb540 118753 lru 00000080:00200000:3.0:1788201707.061477:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000050acda0 118753: 1< 2 - - > 2< 486412288 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70338 - -> 00000080:00200000:3.0:1788201707.061478:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000034bce8e 00000080:00200000:3.0:1788201707.061479:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.061480:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.061485:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.061496:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.061498:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.061499:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.061505:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.061506:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.061508:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.061510:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061514:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.061520:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.061521:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.061523:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.061525:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118753, 118753] 00000080:00200000:3.0:1788201707.061533:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.061535:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.061552:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.061554:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.061564:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486416384, count: 4096 00000080:00200000:3.0:1788201707.061570:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486416384, bytes: 4096 00000080:00200000:3.0:1788201707.061572:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.061574:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.061575:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486416384, 486420480) 0 0 00000080:00200000:3.0:1788201707.061577:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.061580:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118754, 118754] 00020000:00200000:3.0:1788201707.061582:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486416384 chunk: [486416384, 486420480) 486420480, 4096 00020000:00200000:3.0:1788201707.061583:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.061590:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118754, 118754] 00000008:00010000:3.0:1788201707.061593:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.061594:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.061599:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061604:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486416384 486420479) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061609:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061614:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061618:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.061623:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486416384, 486420480) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.061626:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118754 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.061629:0:17463:0:(rw26.c:898:ll_write_end()) pos 486416384, len 4096, copied 4096 00000080:00200000:3.0:1788201707.061631:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000025c303c9[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118754] 00000080:00200000:3.0:1788201707.061633:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000caa64039 fffffc0001029 2:0 ffff93c0c13bb460 118754 lru 00000080:00200000:3.0:1788201707.061637:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a36b5776 118754: 1< 2 - - > 2< 486416384 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70339 - -> 00000080:00200000:3.0:1788201707.061638:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000025c303c9 00000080:00200000:3.0:1788201707.061639:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.061641:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.061646:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.061653:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.061655:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.061656:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.061661:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.061663:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.061665:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.061667:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061672:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.061676:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.061678:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.061680:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.061682:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118754, 118754] 00000080:00200000:3.0:1788201707.061689:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.061691:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.061708:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.061711:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.061721:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486420480, count: 4096 00000080:00200000:3.0:1788201707.061726:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486420480, bytes: 4096 00000080:00200000:3.0:1788201707.061728:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.061730:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.061731:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486420480, 486424576) 0 0 00000080:00200000:3.0:1788201707.061733:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.061736:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118755, 118755] 00020000:00200000:3.0:1788201707.061738:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486420480 chunk: [486420480, 486424576) 486424576, 4096 00020000:00200000:3.0:1788201707.061739:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.061746:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118755, 118755] 00000008:00010000:3.0:1788201707.061749:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.061750:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.061755:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061761:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486420480 486424575) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061766:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061771:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061775:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.061779:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486420480, 486424576) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.061782:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118755 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.061786:0:17463:0:(rw26.c:898:ll_write_end()) pos 486420480, len 4096, copied 4096 00000080:00200000:3.0:1788201707.061788:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008541c175[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118755] 00000080:00200000:3.0:1788201707.061790:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000081a845d1 fffffc0001029 2:0 ffff93c0c13bb0e0 118755 lru 00000080:00200000:3.0:1788201707.061793:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a265210a 118755: 1< 2 - - > 2< 486420480 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70340 - -> 00000080:00200000:3.0:1788201707.061796:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008541c175 00000080:00200000:3.0:1788201707.061797:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.061798:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.061803:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.061810:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.061812:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.061813:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.061818:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.061820:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.061821:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.061824:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061844:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.061850:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.061852:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.061855:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.061856:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118755, 118755] 00000080:00200000:3.0:1788201707.061864:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.061866:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.061884:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.061887:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.061899:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486424576, count: 4096 00000080:00200000:3.0:1788201707.061905:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486424576, bytes: 4096 00000080:00200000:3.0:1788201707.061907:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.061909:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.061910:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486424576, 486428672) 0 0 00000080:00200000:3.0:1788201707.061911:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.061915:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118756, 118756] 00020000:00200000:3.0:1788201707.061916:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486424576 chunk: [486424576, 486428672) 486428672, 4096 00020000:00200000:3.0:1788201707.061918:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.061924:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118756, 118756] 00000008:00010000:3.0:1788201707.061927:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.061929:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.061934:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061939:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486424576 486428671) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061945:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061949:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.061953:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.061958:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486424576, 486428672) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.061960:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118756 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.061964:0:17463:0:(rw26.c:898:ll_write_end()) pos 486424576, len 4096, copied 4096 00000080:00200000:3.0:1788201707.061966:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009d94f5e9[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118756] 00000080:00200000:3.0:1788201707.061968:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a4471026 fffffc0001029 2:0 ffff93c0c13bb700 118756 lru 00000080:00200000:3.0:1788201707.061972:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000074369682 118756: 1< 2 - - > 2< 486424576 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70341 - -> 00000080:00200000:3.0:1788201707.061973:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009d94f5e9 00000080:00200000:3.0:1788201707.061974:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.061975:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.061980:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.061987:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.061989:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.061990:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.061996:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.061998:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.061999:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.062002:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062017:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.062023:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.062025:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.062028:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.062030:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118756, 118756] 00000080:00200000:3.0:1788201707.062037:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.062039:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.062057:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.062059:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.062070:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486428672, count: 4096 00000080:00200000:3.0:1788201707.062075:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486428672, bytes: 4096 00000080:00200000:3.0:1788201707.062077:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.062079:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.062080:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486428672, 486432768) 0 0 00000080:00200000:3.0:1788201707.062082:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.062085:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118757, 118757] 00020000:00200000:3.0:1788201707.062087:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486428672 chunk: [486428672, 486432768) 486432768, 4096 00020000:00200000:3.0:1788201707.062088:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.062095:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118757, 118757] 00000008:00010000:3.0:1788201707.062098:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.062099:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.062104:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062110:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486428672 486432767) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062115:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062120:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062124:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.062127:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486428672, 486432768) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.062130:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118757 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.062134:0:17463:0:(rw26.c:898:ll_write_end()) pos 486428672, len 4096, copied 4096 00000080:00200000:3.0:1788201707.062137:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000049f6e212[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118757] 00000080:00200000:3.0:1788201707.062138:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005a3236c2 fffffc0001029 2:0 ffff93c0c13bb000 118757 lru 00000080:00200000:3.0:1788201707.062142:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e2a31756 118757: 1< 2 - - > 2< 486428672 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70342 - -> 00000080:00200000:3.0:1788201707.062144:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000049f6e212 00000080:00200000:3.0:1788201707.062144:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.062146:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.062151:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.062159:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.062160:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.062162:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.062167:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.062169:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.062170:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.062173:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062177:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.062182:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.062183:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.062185:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.062187:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118757, 118757] 00000080:00200000:3.0:1788201707.062195:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.062196:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.062214:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.062217:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.062226:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486432768, count: 4096 00000080:00200000:3.0:1788201707.062232:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486432768, bytes: 4096 00000080:00200000:3.0:1788201707.062234:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.062236:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.062237:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486432768, 486436864) 0 0 00000080:00200000:3.0:1788201707.062239:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.062242:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118758, 118758] 00020000:00200000:3.0:1788201707.062244:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486432768 chunk: [486432768, 486436864) 486436864, 4096 00020000:00200000:3.0:1788201707.062245:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.062252:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118758, 118758] 00000008:00010000:3.0:1788201707.062255:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.062256:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.062261:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062267:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486432768 486436863) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062272:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062276:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062280:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.062285:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486432768, 486436864) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.062288:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118758 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.062292:0:17463:0:(rw26.c:898:ll_write_end()) pos 486432768, len 4096, copied 4096 00000080:00200000:3.0:1788201707.062294:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000911c8dcc[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118758] 00000080:00200000:3.0:1788201707.062296:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008d204979 fffffc0001029 2:0 ffff93c0c13bba80 118758 lru 00000080:00200000:3.0:1788201707.062300:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ec6b76f9 118758: 1< 2 - - > 2< 486432768 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70343 - -> 00000080:00200000:3.0:1788201707.062301:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000911c8dcc 00000080:00200000:3.0:1788201707.062302:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.062303:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.062308:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.062316:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.062317:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.062318:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.062324:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.062325:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.062327:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.062329:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062334:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.062339:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.062340:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.062342:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.062344:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118758, 118758] 00000080:00200000:3.0:1788201707.062352:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.062353:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.062371:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.062373:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.062383:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486436864, count: 4096 00000080:00200000:3.0:1788201707.062389:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486436864, bytes: 4096 00000080:00200000:3.0:1788201707.062391:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.062393:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.062394:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486436864, 486440960) 0 0 00000080:00200000:3.0:1788201707.062396:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.062399:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118759, 118759] 00020000:00200000:3.0:1788201707.062401:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486436864 chunk: [486436864, 486440960) 486440960, 4096 00020000:00200000:3.0:1788201707.062402:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.062409:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118759, 118759] 00000008:00010000:3.0:1788201707.062412:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.062413:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.062418:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062423:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486436864 486440959) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062429:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062433:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062437:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.062445:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486436864, 486440960) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.062448:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118759 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.062452:0:17463:0:(rw26.c:898:ll_write_end()) pos 486436864, len 4096, copied 4096 00000080:00200000:3.0:1788201707.062454:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ad2fde32[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118759] 00000080:00200000:3.0:1788201707.062456:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000078ee0033 fffffc0001029 2:0 ffff93c0c44172a0 118759 lru 00000080:00200000:3.0:1788201707.062460:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001cf7aba9 118759: 1< 2 - - > 2< 486436864 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70344 - -> 00000080:00200000:3.0:1788201707.062461:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ad2fde32 00000080:00200000:3.0:1788201707.062462:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.062463:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.062468:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.062476:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.062478:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.062479:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.062485:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.062486:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.062488:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.062490:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062495:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.062500:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.062501:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.062504:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.062506:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118759, 118759] 00000080:00200000:3.0:1788201707.062513:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.062515:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.062533:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.062535:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.062545:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486440960, count: 4096 00000080:00200000:3.0:1788201707.062550:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486440960, bytes: 4096 00000080:00200000:3.0:1788201707.062552:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.062554:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.062555:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486440960, 486445056) 0 0 00000080:00200000:3.0:1788201707.062557:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.062560:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118760, 118760] 00020000:00200000:3.0:1788201707.062562:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486440960 chunk: [486440960, 486445056) 486445056, 4096 00020000:00200000:3.0:1788201707.062563:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.062570:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118760, 118760] 00000008:00010000:3.0:1788201707.062573:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.062574:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.062579:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062585:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486440960 486445055) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062590:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062594:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062602:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.062606:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486440960, 486445056) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.062609:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118760 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.062612:0:17463:0:(rw26.c:898:ll_write_end()) pos 486440960, len 4096, copied 4096 00000080:00200000:3.0:1788201707.062614:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000296113f6[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118760] 00000080:00200000:3.0:1788201707.062616:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000eec227ef fffffc0001029 2:0 ffff93c0c4417e00 118760 lru 00000080:00200000:3.0:1788201707.062619:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003f585e03 118760: 1< 2 - - > 2< 486440960 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70345 - -> 00000080:00200000:3.0:1788201707.062620:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000296113f6 00000080:00200000:3.0:1788201707.062621:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.062622:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.062627:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.062635:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.062636:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.062638:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.062643:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.062645:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.062646:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.062649:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062653:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.062661:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.062663:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.062665:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.062667:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118760, 118760] 00000080:00200000:3.0:1788201707.062674:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.062676:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.062693:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.062695:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.062705:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486445056, count: 4096 00000080:00200000:3.0:1788201707.062711:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486445056, bytes: 4096 00000080:00200000:3.0:1788201707.062713:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.062715:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.062716:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486445056, 486449152) 0 0 00000080:00200000:3.0:1788201707.062717:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.062720:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118761, 118761] 00020000:00200000:3.0:1788201707.062722:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486445056 chunk: [486445056, 486449152) 486449152, 4096 00020000:00200000:3.0:1788201707.062724:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.062730:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118761, 118761] 00000008:00010000:3.0:1788201707.062733:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.062735:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.062739:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062745:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486445056 486449151) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062750:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062758:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062762:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.062766:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486445056, 486449152) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.062769:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118761 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.062773:0:17463:0:(rw26.c:898:ll_write_end()) pos 486445056, len 4096, copied 4096 00000080:00200000:3.0:1788201707.062775:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000dbe0a458[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118761] 00000080:00200000:3.0:1788201707.062776:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000e5fd415 fffffc0001029 2:0 ffff93c0c44171c0 118761 lru 00000080:00200000:3.0:1788201707.062780:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000478a1022 118761: 1< 2 - - > 2< 486445056 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70346 - -> 00000080:00200000:3.0:1788201707.062781:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000dbe0a458 00000080:00200000:3.0:1788201707.062781:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.062783:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.062788:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.062795:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.062797:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.062798:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.062803:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.062805:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.062807:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.062809:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062817:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.062821:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.062823:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.062825:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.062826:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118761, 118761] 00000080:00200000:3.0:1788201707.062846:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.062848:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.062866:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.062868:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.062878:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486449152, count: 4096 00000080:00200000:3.0:1788201707.062884:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486449152, bytes: 4096 00000080:00200000:3.0:1788201707.062886:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.062888:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.062889:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486449152, 486453248) 0 0 00000080:00200000:3.0:1788201707.062891:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.062894:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118762, 118762] 00020000:00200000:3.0:1788201707.062896:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486449152 chunk: [486449152, 486453248) 486453248, 4096 00020000:00200000:3.0:1788201707.062897:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.062904:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118762, 118762] 00000008:00010000:3.0:1788201707.062907:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.062908:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.062913:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062919:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486449152 486453247) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062927:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062931:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062935:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.062940:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486449152, 486453248) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.062943:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118762 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.062946:0:17463:0:(rw26.c:898:ll_write_end()) pos 486449152, len 4096, copied 4096 00000080:00200000:3.0:1788201707.062949:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f38b21cc[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118762] 00000080:00200000:3.0:1788201707.062951:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000357048b0 fffffc0001029 2:0 ffff93c0c4417620 118762 lru 00000080:00200000:3.0:1788201707.062954:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007b2bf85e 118762: 1< 2 - - > 2< 486449152 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70347 - -> 00000080:00200000:3.0:1788201707.062956:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f38b21cc 00000080:00200000:3.0:1788201707.062956:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.062958:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.062963:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.062970:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.062972:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.062973:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.062979:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.062981:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.062982:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.062985:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.062990:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.062994:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.062996:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.062998:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.063000:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118762, 118762] 00000080:00200000:3.0:1788201707.063017:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.063020:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.063039:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.063041:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.063051:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486453248, count: 4096 00000080:00200000:3.0:1788201707.063057:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486453248, bytes: 4096 00000080:00200000:3.0:1788201707.063059:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.063061:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.063062:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486453248, 486457344) 0 0 00000080:00200000:3.0:1788201707.063063:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.063067:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118763, 118763] 00020000:00200000:3.0:1788201707.063069:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486453248 chunk: [486453248, 486457344) 486457344, 4096 00020000:00200000:3.0:1788201707.063070:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.063077:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118763, 118763] 00000008:00010000:3.0:1788201707.063080:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.063082:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.063086:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063095:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486453248 486457343) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063101:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063105:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063109:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.063113:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486453248, 486457344) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.063116:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118763 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.063120:0:17463:0:(rw26.c:898:ll_write_end()) pos 486453248, len 4096, copied 4096 00000080:00200000:3.0:1788201707.063122:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b0feb078[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118763] 00000080:00200000:3.0:1788201707.063124:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000de2aea27 fffffc0001029 2:0 ffff93c0c4417d20 118763 lru 00000080:00200000:3.0:1788201707.063128:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007968c0bc 118763: 1< 2 - - > 2< 486453248 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70348 - -> 00000080:00200000:3.0:1788201707.063129:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b0feb078 00000080:00200000:3.0:1788201707.063130:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.063131:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.063136:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.063143:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.063145:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.063146:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.063152:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.063154:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.063155:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.063158:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063162:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.063167:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.063168:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.063170:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.063172:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118763, 118763] 00000080:00200000:3.0:1788201707.063180:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.063181:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.063199:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.063201:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.063211:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486457344, count: 4096 00000080:00200000:3.0:1788201707.063217:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486457344, bytes: 4096 00000080:00200000:3.0:1788201707.063219:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.063221:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.063222:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486457344, 486461440) 0 0 00000080:00200000:3.0:1788201707.063223:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.063226:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118764, 118764] 00020000:00200000:3.0:1788201707.063228:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486457344 chunk: [486457344, 486461440) 486461440, 4096 00020000:00200000:3.0:1788201707.063230:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.063237:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118764, 118764] 00000008:00010000:3.0:1788201707.063240:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.063241:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.063246:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063251:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486457344 486461439) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063257:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063261:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063265:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.063269:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486457344, 486461440) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.063272:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118764 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.063276:0:17463:0:(rw26.c:898:ll_write_end()) pos 486457344, len 4096, copied 4096 00000080:00200000:3.0:1788201707.063278:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002849d17b[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118764] 00000080:00200000:3.0:1788201707.063280:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001084a16d fffffc0001029 2:0 ffff93c0c44179a0 118764 lru 00000080:00200000:3.0:1788201707.063283:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ae19c9f0 118764: 1< 2 - - > 2< 486457344 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70349 - -> 00000080:00200000:3.0:1788201707.063285:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002849d17b 00000080:00200000:3.0:1788201707.063286:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.063287:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.063292:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.063299:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.063301:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.063302:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.063308:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.063310:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.063311:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.063313:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063318:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.063323:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.063324:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.063326:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.063328:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118764, 118764] 00000080:00200000:3.0:1788201707.063335:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.063337:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.063355:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.063357:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.063367:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486461440, count: 4096 00000080:00200000:3.0:1788201707.063373:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486461440, bytes: 4096 00000080:00200000:3.0:1788201707.063374:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.063376:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.063377:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486461440, 486465536) 0 0 00000080:00200000:3.0:1788201707.063379:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.063382:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118765, 118765] 00020000:00200000:3.0:1788201707.063384:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486461440 chunk: [486461440, 486465536) 486465536, 4096 00020000:00200000:3.0:1788201707.063385:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.063392:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118765, 118765] 00000008:00010000:3.0:1788201707.063395:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.063397:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.063401:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063407:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486461440 486465535) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063412:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063416:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063420:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.063424:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486461440, 486465536) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.063427:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118765 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.063430:0:17463:0:(rw26.c:898:ll_write_end()) pos 486461440, len 4096, copied 4096 00000080:00200000:3.0:1788201707.063433:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003c4ce93f[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118765] 00000080:00200000:3.0:1788201707.063434:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f6a962e1 fffffc0001029 2:0 ffff93c0c4417b60 118765 lru 00000080:00200000:3.0:1788201707.063438:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004d21f93e 118765: 1< 2 - - > 2< 486461440 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70350 - -> 00000080:00200000:3.0:1788201707.063439:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003c4ce93f 00000080:00200000:3.0:1788201707.063440:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.063441:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.063446:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.063457:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.063458:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.063460:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.063466:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.063468:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.063469:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.063471:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063476:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.063481:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.063482:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.063484:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.063486:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118765, 118765] 00000080:00200000:3.0:1788201707.063494:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.063496:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.063513:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.063516:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.063525:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486465536, count: 4096 00000080:00200000:3.0:1788201707.063531:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486465536, bytes: 4096 00000080:00200000:3.0:1788201707.063533:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.063535:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.063536:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486465536, 486469632) 0 0 00000080:00200000:3.0:1788201707.063538:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.063541:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118766, 118766] 00020000:00200000:3.0:1788201707.063543:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486465536 chunk: [486465536, 486469632) 486469632, 4096 00020000:00200000:3.0:1788201707.063544:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.063551:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118766, 118766] 00000008:00010000:3.0:1788201707.063554:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.063555:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.063560:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063566:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486465536 486469631) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063571:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063575:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063579:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.063584:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486465536, 486469632) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.063587:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118766 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.063590:0:17463:0:(rw26.c:898:ll_write_end()) pos 486465536, len 4096, copied 4096 00000080:00200000:3.0:1788201707.063593:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000497c684c[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118766] 00000080:00200000:3.0:1788201707.063595:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002b931d8e fffffc0001029 2:0 ffff93c0c4417c40 118766 lru 00000080:00200000:3.0:1788201707.063598:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000485086b3 118766: 1< 2 - - > 2< 486465536 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70351 - -> 00000080:00200000:3.0:1788201707.063600:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000497c684c 00000080:00200000:3.0:1788201707.063600:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.063602:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.063607:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.063615:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.063616:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.063617:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.063623:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.063625:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.063627:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.063629:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063634:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.063638:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.063639:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.063641:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.063643:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118766, 118766] 00000080:00200000:3.0:1788201707.063650:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.063652:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.063670:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.063673:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.063682:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486469632, count: 4096 00000080:00200000:3.0:1788201707.063688:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486469632, bytes: 4096 00000080:00200000:3.0:1788201707.063690:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.063692:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.063693:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486469632, 486473728) 0 0 00000080:00200000:3.0:1788201707.063695:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.063698:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118767, 118767] 00020000:00200000:3.0:1788201707.063700:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486469632 chunk: [486469632, 486473728) 486473728, 4096 00020000:00200000:3.0:1788201707.063701:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.063708:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118767, 118767] 00000008:00010000:3.0:1788201707.063711:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.063712:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.063717:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063723:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486469632 486473727) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063728:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063732:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063736:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.063741:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486469632, 486473728) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.063744:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118767 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.063747:0:17463:0:(rw26.c:898:ll_write_end()) pos 486469632, len 4096, copied 4096 00000080:00200000:3.0:1788201707.063749:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007dab2561[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118767] 00000080:00200000:3.0:1788201707.063751:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000562b0663 fffffc0001029 2:0 ffff93c0c4418c40 118767 lru 00000080:00200000:3.0:1788201707.063755:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000893f9f17 118767: 1< 2 - - > 2< 486469632 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70352 - -> 00000080:00200000:3.0:1788201707.063757:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007dab2561 00000080:00200000:3.0:1788201707.063758:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.063759:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.063764:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.063772:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.063774:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.063775:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.063780:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.063782:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.063784:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.063786:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063790:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.063795:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.063796:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.063798:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.063800:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118767, 118767] 00000080:00200000:3.0:1788201707.063808:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.063810:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.063827:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.063841:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.063852:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486473728, count: 4096 00000080:00200000:3.0:1788201707.063858:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486473728, bytes: 4096 00000080:00200000:3.0:1788201707.063860:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.063862:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.063863:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486473728, 486477824) 0 0 00000080:00200000:3.0:1788201707.063865:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.063868:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118768, 118768] 00020000:00200000:3.0:1788201707.063870:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486473728 chunk: [486473728, 486477824) 486477824, 4096 00020000:00200000:3.0:1788201707.063871:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.063878:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118768, 118768] 00000008:00010000:3.0:1788201707.063881:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.063882:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.063887:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063893:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486473728 486477823) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063898:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063903:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063907:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.063911:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486473728, 486477824) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.063914:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118768 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.063917:0:17463:0:(rw26.c:898:ll_write_end()) pos 486473728, len 4096, copied 4096 00000080:00200000:3.0:1788201707.063920:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b3253440[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118768] 00000080:00200000:3.0:1788201707.063922:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d9619b09 fffffc0001029 2:0 ffff93c0c4418540 118768 lru 00000080:00200000:3.0:1788201707.063925:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000054be1ff1 118768: 1< 2 - - > 2< 486473728 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70353 - -> 00000080:00200000:3.0:1788201707.063927:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b3253440 00000080:00200000:3.0:1788201707.063927:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.063929:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.063934:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.063941:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.063943:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.063944:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.063950:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.063952:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.063953:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.063955:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.063960:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.063965:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.063966:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.063969:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.063970:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118768, 118768] 00000080:00200000:3.0:1788201707.063977:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.063979:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.063997:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.063999:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.064020:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486477824, count: 4096 00000080:00200000:3.0:1788201707.064027:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486477824, bytes: 4096 00000080:00200000:3.0:1788201707.064029:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.064031:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.064032:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486477824, 486481920) 0 0 00000080:00200000:3.0:1788201707.064034:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.064037:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118769, 118769] 00020000:00200000:3.0:1788201707.064039:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486477824 chunk: [486477824, 486481920) 486481920, 4096 00020000:00200000:3.0:1788201707.064041:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.064048:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118769, 118769] 00000008:00010000:3.0:1788201707.064051:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.064052:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.064057:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064063:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486477824 486481919) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064068:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064073:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064077:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.064081:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486477824, 486481920) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.064084:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118769 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.064088:0:17463:0:(rw26.c:898:ll_write_end()) pos 486477824, len 4096, copied 4096 00000080:00200000:3.0:1788201707.064091:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006d2e3191[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118769] 00000080:00200000:3.0:1788201707.064092:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000040ea2321 fffffc0001029 2:0 ffff93c0c4418460 118769 lru 00000080:00200000:3.0:1788201707.064096:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ca84c045 118769: 1< 2 - - > 2< 486477824 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70354 - -> 00000080:00200000:3.0:1788201707.064097:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006d2e3191 00000080:00200000:3.0:1788201707.064098:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.064099:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.064105:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.064113:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.064114:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.064116:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.064121:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.064123:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.064124:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.064127:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064132:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.064136:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.064138:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.064140:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.064141:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118769, 118769] 00000080:00200000:3.0:1788201707.064149:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.064151:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.064169:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.064172:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.064182:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486481920, count: 4096 00000080:00200000:3.0:1788201707.064188:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486481920, bytes: 4096 00000080:00200000:3.0:1788201707.064189:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.064192:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.064193:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486481920, 486486016) 0 0 00000080:00200000:3.0:1788201707.064194:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.064197:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118770, 118770] 00020000:00200000:3.0:1788201707.064199:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486481920 chunk: [486481920, 486486016) 486486016, 4096 00020000:00200000:3.0:1788201707.064201:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.064207:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118770, 118770] 00000008:00010000:3.0:1788201707.064210:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.064212:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.064216:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064222:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486481920 486486015) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064228:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064232:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064236:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.064241:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486481920, 486486016) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.064244:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118770 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.064248:0:17463:0:(rw26.c:898:ll_write_end()) pos 486481920, len 4096, copied 4096 00000080:00200000:3.0:1788201707.064250:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000628c4aea[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118770] 00000080:00200000:3.0:1788201707.064252:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000049b5ef41 fffffc0001029 2:0 ffff93c0c44180e0 118770 lru 00000080:00200000:3.0:1788201707.064255:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f53613e4 118770: 1< 2 - - > 2< 486481920 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70355 - -> 00000080:00200000:3.0:1788201707.064257:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000628c4aea 00000080:00200000:3.0:1788201707.064257:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.064259:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.064264:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.064271:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.064273:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.064274:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.064280:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.064282:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.064284:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.064286:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064291:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.064295:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.064297:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.064299:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.064300:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118770, 118770] 00000080:00200000:3.0:1788201707.064308:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.064310:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.064327:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.064329:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.064339:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486486016, count: 4096 00000080:00200000:3.0:1788201707.064345:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486486016, bytes: 4096 00000080:00200000:3.0:1788201707.064347:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.064349:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.064351:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486486016, 486490112) 0 0 00000080:00200000:3.0:1788201707.064352:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.064355:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118771, 118771] 00020000:00200000:3.0:1788201707.064357:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486486016 chunk: [486486016, 486490112) 486490112, 4096 00020000:00200000:3.0:1788201707.064358:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.064365:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118771, 118771] 00000008:00010000:3.0:1788201707.064368:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.064369:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.064374:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064380:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486486016 486490111) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064385:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064389:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064393:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.064401:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486486016, 486490112) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.064404:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118771 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.064407:0:17463:0:(rw26.c:898:ll_write_end()) pos 486486016, len 4096, copied 4096 00000080:00200000:3.0:1788201707.064409:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e55cc9b1[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118771] 00000080:00200000:3.0:1788201707.064411:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000087ffc4c0 fffffc0001029 2:0 ffff93c0c4418700 118771 lru 00000080:00200000:3.0:1788201707.064415:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000f28829e 118771: 1< 2 - - > 2< 486486016 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70356 - -> 00000080:00200000:3.0:1788201707.064417:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e55cc9b1 00000080:00200000:3.0:1788201707.064417:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.064418:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.064423:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.064431:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.064432:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.064434:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.064439:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.064441:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.064443:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.064445:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064450:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.064455:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.064456:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.064459:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.064460:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118771, 118771] 00000080:00200000:3.0:1788201707.064468:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.064470:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.064488:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.064490:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.064500:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486490112, count: 4096 00000080:00200000:3.0:1788201707.064506:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486490112, bytes: 4096 00000080:00200000:3.0:1788201707.064508:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.064510:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.064511:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486490112, 486494208) 0 0 00000080:00200000:3.0:1788201707.064512:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.064515:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118772, 118772] 00020000:00200000:3.0:1788201707.064517:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486490112 chunk: [486490112, 486494208) 486494208, 4096 00020000:00200000:3.0:1788201707.064519:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.064525:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118772, 118772] 00000008:00010000:3.0:1788201707.064528:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.064529:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.064534:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064540:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486490112 486494207) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064545:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064550:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064557:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.064561:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486490112, 486494208) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.064564:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118772 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.064567:0:17463:0:(rw26.c:898:ll_write_end()) pos 486490112, len 4096, copied 4096 00000080:00200000:3.0:1788201707.064570:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001e843729[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118772] 00000080:00200000:3.0:1788201707.064571:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000090657d5 fffffc0001029 2:0 ffff93c0c4418000 118772 lru 00000080:00200000:3.0:1788201707.064575:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006feb0d44 118772: 1< 2 - - > 2< 486490112 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70357 - -> 00000080:00200000:3.0:1788201707.064576:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001e843729 00000080:00200000:3.0:1788201707.064576:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.064578:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.064583:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.064590:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.064592:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.064593:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.064599:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.064600:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.064602:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.064604:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064609:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.064617:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.064619:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.064621:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.064623:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118772, 118772] 00000080:00200000:3.0:1788201707.064630:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.064632:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.064650:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.064652:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.064662:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486494208, count: 4096 00000080:00200000:3.0:1788201707.064667:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486494208, bytes: 4096 00000080:00200000:3.0:1788201707.064669:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.064671:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.064672:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486494208, 486498304) 0 0 00000080:00200000:3.0:1788201707.064674:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.064677:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118773, 118773] 00020000:00200000:3.0:1788201707.064679:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486494208 chunk: [486494208, 486498304) 486498304, 4096 00020000:00200000:3.0:1788201707.064681:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.064687:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118773, 118773] 00000008:00010000:3.0:1788201707.064690:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.064691:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.064696:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064702:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486494208 486498303) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064707:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064715:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064719:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.064723:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486494208, 486498304) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.064726:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118773 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.064729:0:17463:0:(rw26.c:898:ll_write_end()) pos 486494208, len 4096, copied 4096 00000080:00200000:3.0:1788201707.064732:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d70bb361[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118773] 00000080:00200000:3.0:1788201707.064733:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001f7edfee fffffc0001029 2:0 ffff93c0c4418a80 118773 lru 00000080:00200000:3.0:1788201707.064737:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b6e3a29b 118773: 1< 2 - - > 2< 486494208 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70358 - -> 00000080:00200000:3.0:1788201707.064738:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d70bb361 00000080:00200000:3.0:1788201707.064738:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.064739:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.064745:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.064752:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.064754:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.064755:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.064760:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.064762:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.064764:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.064766:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064774:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.064778:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.064780:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.064782:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.064783:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118773, 118773] 00000080:00200000:3.0:1788201707.064791:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.064793:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.064810:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.064812:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.064822:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486498304, count: 4096 00000080:00200000:3.0:1788201707.064840:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486498304, bytes: 4096 00000080:00200000:3.0:1788201707.064842:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.064844:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.064845:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486498304, 486502400) 0 0 00000080:00200000:3.0:1788201707.064847:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.064850:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118774, 118774] 00020000:00200000:3.0:1788201707.064852:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486498304 chunk: [486498304, 486502400) 486502400, 4096 00020000:00200000:3.0:1788201707.064853:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.064861:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118774, 118774] 00000008:00010000:3.0:1788201707.064864:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.064865:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.064870:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064876:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486498304 486502399) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064885:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064889:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064893:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.064898:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486498304, 486502400) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.064901:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118774 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.064904:0:17463:0:(rw26.c:898:ll_write_end()) pos 486498304, len 4096, copied 4096 00000080:00200000:3.0:1788201707.064907:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000074a60a3f[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118774] 00000080:00200000:3.0:1788201707.064908:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e199c5e1 fffffc0001029 2:0 ffff93c0c4418380 118774 lru 00000080:00200000:3.0:1788201707.064912:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ee89ac2f 118774: 1< 2 - - > 2< 486498304 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70359 - -> 00000080:00200000:3.0:1788201707.064913:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000074a60a3f 00000080:00200000:3.0:1788201707.064914:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.064915:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.064921:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.064928:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.064930:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.064931:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.064937:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.064939:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.064941:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.064943:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.064948:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.064953:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.064954:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.064956:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.064958:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118774, 118774] 00000080:00200000:3.0:1788201707.064965:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.064967:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.064985:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.064987:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.064997:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486502400, count: 4096 00000080:00200000:3.0:1788201707.065003:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486502400, bytes: 4096 00000080:00200000:3.0:1788201707.065005:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.065017:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.065019:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486502400, 486506496) 0 0 00000080:00200000:3.0:1788201707.065020:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.065024:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118775, 118775] 00020000:00200000:3.0:1788201707.065026:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486502400 chunk: [486502400, 486506496) 486506496, 4096 00020000:00200000:3.0:1788201707.065027:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.065035:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118775, 118775] 00000008:00010000:3.0:1788201707.065038:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.065039:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.065044:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065053:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486502400 486506495) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065058:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065063:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065067:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.065071:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486502400, 486506496) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.065074:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118775 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.065078:0:17463:0:(rw26.c:898:ll_write_end()) pos 486502400, len 4096, copied 4096 00000080:00200000:3.0:1788201707.065080:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003282eb38[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118775] 00000080:00200000:3.0:1788201707.065082:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000fbfc8d7 fffffc0001029 2:0 ffff93c0c44188c0 118775 lru 00000080:00200000:3.0:1788201707.065086:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000649dbd3e 118775: 1< 2 - - > 2< 486502400 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70360 - -> 00000080:00200000:3.0:1788201707.065087:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003282eb38 00000080:00200000:3.0:1788201707.065088:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.065089:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.065094:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.065102:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.065104:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.065105:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.065111:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.065113:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.065114:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.065117:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065121:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.065126:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.065127:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.065130:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.065131:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118775, 118775] 00000080:00200000:3.0:1788201707.065139:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.065141:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.065159:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.065161:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.065171:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486506496, count: 4096 00000080:00200000:3.0:1788201707.065177:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486506496, bytes: 4096 00000080:00200000:3.0:1788201707.065179:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.065181:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.065182:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486506496, 486510592) 0 0 00000080:00200000:3.0:1788201707.065183:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.065186:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118776, 118776] 00020000:00200000:3.0:1788201707.065188:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486506496 chunk: [486506496, 486510592) 486510592, 4096 00020000:00200000:3.0:1788201707.065190:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.065196:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118776, 118776] 00000008:00010000:3.0:1788201707.065199:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.065201:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.065206:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065211:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486506496 486510591) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065217:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065221:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065225:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.065230:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486506496, 486510592) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.065233:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118776 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.065236:0:17463:0:(rw26.c:898:ll_write_end()) pos 486506496, len 4096, copied 4096 00000080:00200000:3.0:1788201707.065239:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d9c43b21[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118776] 00000080:00200000:3.0:1788201707.065241:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a2639d9a fffffc0001029 2:0 ffff93c0c4418ee0 118776 lru 00000080:00200000:3.0:1788201707.065244:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000654b4352 118776: 1< 2 - - > 2< 486506496 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70361 - -> 00000080:00200000:3.0:1788201707.065246:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d9c43b21 00000080:00200000:3.0:1788201707.065246:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.065248:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.065253:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.065260:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.065262:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.065263:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.065269:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.065270:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.065272:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.065274:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065279:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.065284:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.065285:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.065288:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.065289:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118776, 118776] 00000080:00200000:3.0:1788201707.065297:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.065299:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.065316:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.065318:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.065328:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486510592, count: 4096 00000080:00200000:3.0:1788201707.065334:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486510592, bytes: 4096 00000080:00200000:3.0:1788201707.065336:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.065338:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.065339:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486510592, 486514688) 0 0 00000080:00200000:3.0:1788201707.065340:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.065344:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118777, 118777] 00020000:00200000:3.0:1788201707.065346:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486510592 chunk: [486510592, 486514688) 486514688, 4096 00020000:00200000:3.0:1788201707.065347:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.065353:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118777, 118777] 00000008:00010000:3.0:1788201707.065357:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.065358:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.065363:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065368:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486510592 486514687) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065374:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065378:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065382:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.065387:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486510592, 486514688) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.065390:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118777 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.065393:0:17463:0:(rw26.c:898:ll_write_end()) pos 486510592, len 4096, copied 4096 00000080:00200000:3.0:1788201707.065395:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002c65174c[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118777] 00000080:00200000:3.0:1788201707.065397:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000039921154 fffffc0001029 2:0 ffff93c0c44187e0 118777 lru 00000080:00200000:3.0:1788201707.065401:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001e6bdf6c 118777: 1< 2 - - > 2< 486510592 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70362 - -> 00000080:00200000:3.0:1788201707.065402:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002c65174c 00000080:00200000:3.0:1788201707.065403:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.065404:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.065409:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.065420:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.065421:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.065423:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.065428:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.065430:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.065431:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.065434:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065439:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.065444:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.065445:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.065447:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.065449:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118777, 118777] 00000080:00200000:3.0:1788201707.065456:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.065458:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.065476:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.065478:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.065487:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486514688, count: 4096 00000080:00200000:3.0:1788201707.065493:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486514688, bytes: 4096 00000080:00200000:3.0:1788201707.065495:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.065497:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.065498:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486514688, 486518784) 0 0 00000080:00200000:3.0:1788201707.065500:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.065503:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118778, 118778] 00020000:00200000:3.0:1788201707.065505:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486514688 chunk: [486514688, 486518784) 486518784, 4096 00020000:00200000:3.0:1788201707.065507:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.065513:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118778, 118778] 00000008:00010000:3.0:1788201707.065516:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.065518:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.065522:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065528:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486514688 486518783) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065533:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065538:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065541:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.065546:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486514688, 486518784) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.065549:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118778 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.065553:0:17463:0:(rw26.c:898:ll_write_end()) pos 486514688, len 4096, copied 4096 00000080:00200000:3.0:1788201707.065555:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000531068b2[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118778] 00000080:00200000:3.0:1788201707.065557:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d7416ee4 fffffc0001029 2:0 ffff93c0c44182a0 118778 lru 00000080:00200000:3.0:1788201707.065560:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003061fd27 118778: 1< 2 - - > 2< 486514688 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70363 - -> 00000080:00200000:3.0:1788201707.065562:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000531068b2 00000080:00200000:3.0:1788201707.065563:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.065564:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.065569:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.065577:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.065578:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.065579:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.065585:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.065587:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.065589:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.065591:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065596:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.065600:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.065602:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.065604:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.065606:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118778, 118778] 00000080:00200000:3.0:1788201707.065613:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.065614:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.065632:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.065634:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.065644:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486518784, count: 4096 00000080:00200000:3.0:1788201707.065650:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486518784, bytes: 4096 00000080:00200000:3.0:1788201707.065652:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.065654:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.065655:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486518784, 486522880) 0 0 00000080:00200000:3.0:1788201707.065656:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.065660:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118779, 118779] 00020000:00200000:3.0:1788201707.065661:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486518784 chunk: [486518784, 486522880) 486522880, 4096 00020000:00200000:3.0:1788201707.065663:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.065669:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118779, 118779] 00000008:00010000:3.0:1788201707.065672:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.065674:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.065679:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065684:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486518784 486522879) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065690:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065694:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065698:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.065703:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486518784, 486522880) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.065706:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118779 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.065709:0:17463:0:(rw26.c:898:ll_write_end()) pos 486518784, len 4096, copied 4096 00000080:00200000:3.0:1788201707.065712:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000063056139[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118779] 00000080:00200000:3.0:1788201707.065714:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003bbfd9f5 fffffc0001029 2:0 ffff93c0c4418e00 118779 lru 00000080:00200000:3.0:1788201707.065717:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000712aacdf 118779: 1< 2 - - > 2< 486518784 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70364 - -> 00000080:00200000:3.0:1788201707.065719:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000063056139 00000080:00200000:3.0:1788201707.065720:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.065721:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.065727:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.065734:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.065736:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.065737:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.065742:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.065744:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.065746:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.065748:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065752:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.065756:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.065758:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.065760:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.065762:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118779, 118779] 00000080:00200000:3.0:1788201707.065769:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.065771:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.065789:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.065791:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.065800:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486522880, count: 4096 00000080:00200000:3.0:1788201707.065806:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486522880, bytes: 4096 00000080:00200000:3.0:1788201707.065808:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.065811:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.065812:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486522880, 486526976) 0 0 00000080:00200000:3.0:1788201707.065813:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.065816:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118780, 118780] 00020000:00200000:3.0:1788201707.065818:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486522880 chunk: [486522880, 486526976) 486526976, 4096 00020000:00200000:3.0:1788201707.065819:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.065826:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118780, 118780] 00000008:00010000:3.0:1788201707.065841:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.065843:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.065848:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065854:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486522880 486526975) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065859:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065864:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065868:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.065872:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486522880, 486526976) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.065875:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118780 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.065879:0:17463:0:(rw26.c:898:ll_write_end()) pos 486522880, len 4096, copied 4096 00000080:00200000:3.0:1788201707.065881:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000068d5d290[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118780] 00000080:00200000:3.0:1788201707.065883:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e4864a6a fffffc0001029 2:0 ffff93c0c44181c0 118780 lru 00000080:00200000:3.0:1788201707.065887:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008d7a27eb 118780: 1< 2 - - > 2< 486522880 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70365 - -> 00000080:00200000:3.0:1788201707.065888:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000068d5d290 00000080:00200000:3.0:1788201707.065889:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.065890:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.065895:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.065903:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.065904:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.065905:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.065911:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.065913:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.065914:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.065917:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.065921:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.065926:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.065928:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.065930:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.065931:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118780, 118780] 00000080:00200000:3.0:1788201707.065939:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.065941:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.065958:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.065960:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.065971:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486526976, count: 4096 00000080:00200000:3.0:1788201707.065977:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486526976, bytes: 4096 00000080:00200000:3.0:1788201707.065979:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.065981:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.065982:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486526976, 486531072) 0 0 00000080:00200000:3.0:1788201707.065984:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.065987:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118781, 118781] 00020000:00200000:3.0:1788201707.065989:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486526976 chunk: [486526976, 486531072) 486531072, 4096 00020000:00200000:3.0:1788201707.065990:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.065997:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118781, 118781] 00000008:00010000:3.0:1788201707.066000:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.066001:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.066006:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066022:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486526976 486531071) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066028:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066032:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066037:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.066041:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486526976, 486531072) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.066044:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118781 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.066048:0:17463:0:(rw26.c:898:ll_write_end()) pos 486526976, len 4096, copied 4096 00000080:00200000:3.0:1788201707.066050:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006ef5758a[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118781] 00000080:00200000:3.0:1788201707.066052:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d3110e0d fffffc0001029 2:0 ffff93c0c4418620 118781 lru 00000080:00200000:3.0:1788201707.066055:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004c99ffb6 118781: 1< 2 - - > 2< 486526976 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70366 - -> 00000080:00200000:3.0:1788201707.066057:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006ef5758a 00000080:00200000:3.0:1788201707.066058:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.066059:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.066065:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.066072:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.066074:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.066075:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.066081:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.066083:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.066084:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.066087:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066092:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.066096:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.066098:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.066100:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.066102:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118781, 118781] 00000080:00200000:3.0:1788201707.066109:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.066111:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.066129:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.066132:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.066141:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486531072, count: 4096 00000080:00200000:3.0:1788201707.066147:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486531072, bytes: 4096 00000080:00200000:3.0:1788201707.066149:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.066151:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.066152:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486531072, 486535168) 0 0 00000080:00200000:3.0:1788201707.066154:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.066157:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118782, 118782] 00020000:00200000:3.0:1788201707.066159:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486531072 chunk: [486531072, 486535168) 486535168, 4096 00020000:00200000:3.0:1788201707.066161:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.066167:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118782, 118782] 00000008:00010000:3.0:1788201707.066170:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.066172:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.066176:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066182:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486531072 486535167) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066188:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066192:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066196:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.066200:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486531072, 486535168) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.066203:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118782 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.066207:0:17463:0:(rw26.c:898:ll_write_end()) pos 486531072, len 4096, copied 4096 00000080:00200000:3.0:1788201707.066209:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ccdeb113[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118782] 00000080:00200000:3.0:1788201707.066211:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006108db5b fffffc0001029 2:0 ffff93c0c4418d20 118782 lru 00000080:00200000:3.0:1788201707.066215:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ace2c3f1 118782: 1< 2 - - > 2< 486531072 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70367 - -> 00000080:00200000:3.0:1788201707.066216:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ccdeb113 00000080:00200000:3.0:1788201707.066217:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.066218:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.066223:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.066230:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.066232:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.066233:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.066239:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.066241:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.066242:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.066245:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066249:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.066254:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.066255:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.066258:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.066260:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118782, 118782] 00000080:00200000:3.0:1788201707.066267:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.066269:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.066287:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.066289:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.066299:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486535168, count: 4096 00000080:00200000:3.0:1788201707.066305:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486535168, bytes: 4096 00000080:00200000:3.0:1788201707.066306:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.066308:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.066309:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486535168, 486539264) 0 0 00000080:00200000:3.0:1788201707.066311:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.066314:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118783, 118783] 00020000:00200000:3.0:1788201707.066316:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486535168 chunk: [486535168, 486539264) 486539264, 4096 00020000:00200000:3.0:1788201707.066318:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.066324:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118783, 118783] 00000008:00010000:3.0:1788201707.066327:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.066329:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.066333:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066339:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486535168 486539263) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066344:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066349:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066352:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.066360:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486535168, 486539264) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.066363:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118783 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.066367:0:17463:0:(rw26.c:898:ll_write_end()) pos 486535168, len 4096, copied 4096 00000080:00200000:3.0:1788201707.066369:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ac44add9[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118783] 00000080:00200000:3.0:1788201707.066371:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000077a1b693 fffffc0001029 2:0 ffff93c0c44189a0 118783 lru 00000080:00200000:3.0:1788201707.066375:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008404362f 118783: 1< 2 - - > 2< 486535168 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70368 - -> 00000080:00200000:3.0:1788201707.066376:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ac44add9 00000080:00200000:3.0:1788201707.066377:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.066378:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.066383:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000090290fb6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.066391:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.066392:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.066393:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.066399:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.066400:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.066402:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.066404:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066409:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.066414:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.066415:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.066418:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.066420:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118783, 118783] 00000080:00200000:3.0:1788201707.066427:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.066429:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.066446:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.066449:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.066458:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486539264, count: 4096 00000080:00200000:3.0:1788201707.066464:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486539264, bytes: 4096 00000080:00200000:3.0:1788201707.066466:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.066468:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.066469:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486539264, 486543360) 0 0 00000080:00200000:3.0:1788201707.066471:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.066474:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118784, 118784] 00020000:00200000:3.0:1788201707.066476:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486539264 chunk: [486539264, 486543360) 486543360, 4096 00020000:00200000:3.0:1788201707.066477:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.066484:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118784, 118784] 00000008:00010000:3.0:1788201707.066487:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.066488:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.066493:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066498:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486539264 486543359) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 286/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066504:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066508:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066516:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.066520:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486539264, 486543360) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.066523:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118784 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.066526:0:17463:0:(rw26.c:898:ll_write_end()) pos 486539264, len 4096, copied 4096 00000080:00200000:3.0:1788201707.066528:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000273fd245[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118784] 00000080:00200000:3.0:1788201707.066530:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000063ab9451 fffffc0001029 2:0 ffff93c0c13bb380 118784 lru 00000080:00200000:3.0:1788201707.066533:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001b93c589 118784: 1< 2 - - > 2< 486539264 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 3407872 - | + - + - > 5< + - + - | 0 - | 70369 - -> 00000080:00200000:3.0:1788201707.066535:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000273fd245 00000080:00200000:3.0:1788201707.066535:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.066537:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.066543:0:17463:0:(osc_cache.c:857:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.066549:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.066551:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.066552:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.066558:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.066560:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.066561:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.066564:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066569:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.066577:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.066578:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.066581:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.066583:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118784, 118784] 00000080:00200000:3.0:1788201707.066590:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.066592:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.066610:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.066612:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.066622:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486543360, count: 4096 00000080:00200000:3.0:1788201707.066628:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486543360, bytes: 4096 00000080:00200000:3.0:1788201707.066630:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.066632:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.066633:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486543360, 486547456) 0 0 00000080:00200000:3.0:1788201707.066634:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.066637:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118785, 118785] 00020000:00200000:3.0:1788201707.066639:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486543360 chunk: [486543360, 486547456) 486547456, 4096 00020000:00200000:3.0:1788201707.066640:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.066646:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118785, 118785] 00000008:00010000:3.0:1788201707.066649:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.066650:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.066655:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066661:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486543360 486547455) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066667:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066675:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066679:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.066683:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486543360, 486547456) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.066686:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118785 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.066689:0:17463:0:(rw26.c:898:ll_write_end()) pos 486543360, len 4096, copied 4096 00000080:00200000:3.0:1788201707.066691:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000393d7203[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118785] 00000080:00200000:3.0:1788201707.066693:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009db44088 fffffc0001029 2:0 ffff93c0c13bb8c0 118785 lru 00000080:00200000:3.0:1788201707.066696:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004cb71340 118785: 1< 2 - - > 2< 486543360 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70370 - -> 00000080:00200000:3.0:1788201707.066697:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000393d7203 00000080:00200000:3.0:1788201707.066698:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.066699:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.066705:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.066712:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.066714:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.066715:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.066721:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.066722:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.066724:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.066726:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066734:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.066739:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.066740:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.066742:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.066744:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118785, 118785] 00000080:00200000:3.0:1788201707.066751:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.066753:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.066771:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.066773:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.066782:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486547456, count: 4096 00000080:00200000:3.0:1788201707.066788:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486547456, bytes: 4096 00000080:00200000:3.0:1788201707.066790:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.066792:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.066793:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486547456, 486551552) 0 0 00000080:00200000:3.0:1788201707.066795:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.066798:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118786, 118786] 00020000:00200000:3.0:1788201707.066800:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486547456 chunk: [486547456, 486551552) 486551552, 4096 00020000:00200000:3.0:1788201707.066801:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.066807:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118786, 118786] 00000008:00010000:3.0:1788201707.066810:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.066812:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.066816:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066822:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486547456 486551551) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066843:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066848:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066852:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.066856:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486547456, 486551552) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.066859:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118786 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.066863:0:17463:0:(rw26.c:898:ll_write_end()) pos 486547456, len 4096, copied 4096 00000080:00200000:3.0:1788201707.066865:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e3a79984[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118786] 00000080:00200000:3.0:1788201707.066866:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d8d3cfc7 fffffc0001029 2:0 ffff93c0c13bbee0 118786 lru 00000080:00200000:3.0:1788201707.066870:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000099454832 118786: 1< 2 - - > 2< 486547456 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70371 - -> 00000080:00200000:3.0:1788201707.066871:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e3a79984 00000080:00200000:3.0:1788201707.066872:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.066873:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.066879:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.066887:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.066888:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.066889:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.066895:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.066897:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.066899:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.066901:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066906:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.066911:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.066912:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.066915:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.066916:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118786, 118786] 00000080:00200000:3.0:1788201707.066924:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.066926:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.066943:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.066945:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.066955:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486551552, count: 4096 00000080:00200000:3.0:1788201707.066961:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486551552, bytes: 4096 00000080:00200000:3.0:1788201707.066963:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.066965:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.066966:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486551552, 486555648) 0 0 00000080:00200000:3.0:1788201707.066967:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.066971:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118787, 118787] 00020000:00200000:3.0:1788201707.066972:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486551552 chunk: [486551552, 486555648) 486555648, 4096 00020000:00200000:3.0:1788201707.066974:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.066980:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118787, 118787] 00000008:00010000:3.0:1788201707.066983:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.066985:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.066990:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.066998:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486551552 486555647) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067004:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067019:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067023:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.067028:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486551552, 486555648) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.067031:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118787 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.067035:0:17463:0:(rw26.c:898:ll_write_end()) pos 486551552, len 4096, copied 4096 00000080:00200000:3.0:1788201707.067037:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000cff76c5[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118787] 00000080:00200000:3.0:1788201707.067039:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c12b5a69 fffffc0001029 2:0 ffff93c0c13bb7e0 118787 lru 00000080:00200000:3.0:1788201707.067043:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000535d705f 118787: 1< 2 - - > 2< 486551552 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70372 - -> 00000080:00200000:3.0:1788201707.067044:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000cff76c5 00000080:00200000:3.0:1788201707.067045:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.067046:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.067052:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.067059:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.067061:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.067062:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.067068:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.067070:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.067071:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.067074:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067079:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.067084:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.067085:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.067087:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.067089:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118787, 118787] 00000080:00200000:3.0:1788201707.067097:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.067098:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.067116:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.067118:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.067128:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486555648, count: 4096 00000080:00200000:3.0:1788201707.067134:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486555648, bytes: 4096 00000080:00200000:3.0:1788201707.067136:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.067138:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.067139:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486555648, 486559744) 0 0 00000080:00200000:3.0:1788201707.067140:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.067144:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118788, 118788] 00020000:00200000:3.0:1788201707.067146:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486555648 chunk: [486555648, 486559744) 486559744, 4096 00020000:00200000:3.0:1788201707.067147:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.067154:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118788, 118788] 00000008:00010000:3.0:1788201707.067157:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.067158:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.067163:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067169:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486555648 486559743) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067174:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067178:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067182:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.067187:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486555648, 486559744) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.067190:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118788 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.067193:0:17463:0:(rw26.c:898:ll_write_end()) pos 486555648, len 4096, copied 4096 00000080:00200000:3.0:1788201707.067195:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006bc16cd0[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118788] 00000080:00200000:3.0:1788201707.067197:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006f30959a fffffc0001029 2:0 ffff93c0c13bb2a0 118788 lru 00000080:00200000:3.0:1788201707.067200:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005b19dc97 118788: 1< 2 - - > 2< 486555648 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70373 - -> 00000080:00200000:3.0:1788201707.067202:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006bc16cd0 00000080:00200000:3.0:1788201707.067203:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.067204:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.067209:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.067216:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.067218:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.067220:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.067225:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.067227:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.067228:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.067231:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067235:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.067240:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.067241:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.067244:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.067245:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118788, 118788] 00000080:00200000:3.0:1788201707.067253:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.067255:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.067272:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.067274:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.067284:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486559744, count: 4096 00000080:00200000:3.0:1788201707.067290:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486559744, bytes: 4096 00000080:00200000:3.0:1788201707.067292:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.067294:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.067295:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486559744, 486563840) 0 0 00000080:00200000:3.0:1788201707.067296:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.067300:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118789, 118789] 00020000:00200000:3.0:1788201707.067301:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486559744 chunk: [486559744, 486563840) 486563840, 4096 00020000:00200000:3.0:1788201707.067303:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.067309:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118789, 118789] 00000008:00010000:3.0:1788201707.067312:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.067314:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.067319:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067324:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486559744 486563839) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067330:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067334:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067338:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.067342:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486559744, 486563840) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.067344:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118789 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.067348:0:17463:0:(rw26.c:898:ll_write_end()) pos 486559744, len 4096, copied 4096 00000080:00200000:3.0:1788201707.067350:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c16505f6[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118789] 00000080:00200000:3.0:1788201707.067352:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000da93baea fffffc0001029 2:0 ffff93c0c13bbe00 118789 lru 00000080:00200000:3.0:1788201707.067355:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009b27d49a 118789: 1< 2 - - > 2< 486559744 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70374 - -> 00000080:00200000:3.0:1788201707.067357:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c16505f6 00000080:00200000:3.0:1788201707.067357:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.067359:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.067364:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.067374:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.067376:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.067377:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.067383:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.067385:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.067386:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.067389:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067393:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.067399:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.067400:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.067402:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.067404:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118789, 118789] 00000080:00200000:3.0:1788201707.067412:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.067414:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.067431:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.067433:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.067443:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486563840, count: 4096 00000080:00200000:3.0:1788201707.067449:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486563840, bytes: 4096 00000080:00200000:3.0:1788201707.067451:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.067453:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.067454:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486563840, 486567936) 0 0 00000080:00200000:3.0:1788201707.067456:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.067459:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118790, 118790] 00020000:00200000:3.0:1788201707.067461:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486563840 chunk: [486563840, 486567936) 486567936, 4096 00020000:00200000:3.0:1788201707.067463:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.067469:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118790, 118790] 00000008:00010000:3.0:1788201707.067472:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.067474:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.067478:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067484:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486563840 486567935) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067489:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067494:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067497:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.067502:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486563840, 486567936) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.067505:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118790 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.067509:0:17463:0:(rw26.c:898:ll_write_end()) pos 486563840, len 4096, copied 4096 00000080:00200000:3.0:1788201707.067511:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002cdc3819[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118790] 00000080:00200000:3.0:1788201707.067513:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008cf01cc7 fffffc0001029 2:0 ffff93c0c13bb1c0 118790 lru 00000080:00200000:3.0:1788201707.067516:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000044aa3713 118790: 1< 2 - - > 2< 486563840 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70375 - -> 00000080:00200000:3.0:1788201707.067518:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002cdc3819 00000080:00200000:3.0:1788201707.067518:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.067520:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.067525:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.067533:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.067534:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.067535:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.067541:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.067543:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.067544:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.067547:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067551:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.067556:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.067557:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.067559:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.067561:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118790, 118790] 00000080:00200000:3.0:1788201707.067568:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.067570:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.067587:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.067590:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.067599:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486567936, count: 4096 00000080:00200000:3.0:1788201707.067605:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486567936, bytes: 4096 00000080:00200000:3.0:1788201707.067607:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.067609:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.067610:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486567936, 486572032) 0 0 00000080:00200000:3.0:1788201707.067612:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.067615:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118791, 118791] 00020000:00200000:3.0:1788201707.067617:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486567936 chunk: [486567936, 486572032) 486572032, 4096 00020000:00200000:3.0:1788201707.067618:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.067625:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118791, 118791] 00000008:00010000:3.0:1788201707.067628:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.067629:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.067634:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067639:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486567936 486572031) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067645:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067649:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067653:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.067658:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486567936, 486572032) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.067661:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118791 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.067664:0:17463:0:(rw26.c:898:ll_write_end()) pos 486567936, len 4096, copied 4096 00000080:00200000:3.0:1788201707.067666:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000053bdd853[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118791] 00000080:00200000:3.0:1788201707.067668:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e40e0fc7 fffffc0001029 2:0 ffff93c0c13bb620 118791 lru 00000080:00200000:3.0:1788201707.067671:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000002cc7da1 118791: 1< 2 - - > 2< 486567936 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70376 - -> 00000080:00200000:3.0:1788201707.067674:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000053bdd853 00000080:00200000:3.0:1788201707.067675:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.067676:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.067682:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.067689:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.067691:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.067692:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.067698:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.067700:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.067701:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.067704:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067708:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.067713:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.067714:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.067716:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.067718:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118791, 118791] 00000080:00200000:3.0:1788201707.067726:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.067728:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.067745:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.067747:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.067757:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486572032, count: 4096 00000080:00200000:3.0:1788201707.067763:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486572032, bytes: 4096 00000080:00200000:3.0:1788201707.067765:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.067767:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.067768:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486572032, 486576128) 0 0 00000080:00200000:3.0:1788201707.067770:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.067773:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118792, 118792] 00020000:00200000:3.0:1788201707.067775:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486572032 chunk: [486572032, 486576128) 486576128, 4096 00020000:00200000:3.0:1788201707.067776:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.067782:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118792, 118792] 00000008:00010000:3.0:1788201707.067785:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.067787:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.067791:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067797:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486572032 486576127) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067803:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067807:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067811:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.067815:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486572032, 486576128) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.067818:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118792 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.067821:0:17463:0:(rw26.c:898:ll_write_end()) pos 486572032, len 4096, copied 4096 00000080:00200000:3.0:1788201707.067824:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000013b06ff[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118792] 00000080:00200000:3.0:1788201707.067826:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002d71ca78 fffffc0001029 2:0 ffff93c0c13bbd20 118792 lru 00000080:00200000:3.0:1788201707.067842:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f4380042 118792: 1< 2 - - > 2< 486572032 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70377 - -> 00000080:00200000:3.0:1788201707.067843:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000013b06ff 00000080:00200000:3.0:1788201707.067844:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.067845:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.067851:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.067859:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.067861:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.067862:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.067868:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.067870:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.067871:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.067874:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067878:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.067883:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.067884:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.067887:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.067889:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118792, 118792] 00000080:00200000:3.0:1788201707.067896:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.067897:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.067915:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.067917:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.067928:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486576128, count: 4096 00000080:00200000:3.0:1788201707.067934:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486576128, bytes: 4096 00000080:00200000:3.0:1788201707.067936:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.067938:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.067939:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486576128, 486580224) 0 0 00000080:00200000:3.0:1788201707.067940:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.067943:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118793, 118793] 00020000:00200000:3.0:1788201707.067945:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486576128 chunk: [486576128, 486580224) 486580224, 4096 00020000:00200000:3.0:1788201707.067946:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.067953:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118793, 118793] 00000008:00010000:3.0:1788201707.067956:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.067958:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.067962:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067968:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486576128 486580223) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067973:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067978:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.067982:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.067985:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486576128, 486580224) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.067988:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118793 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.067992:0:17463:0:(rw26.c:898:ll_write_end()) pos 486576128, len 4096, copied 4096 00000080:00200000:3.0:1788201707.067995:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003e20841c[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118793] 00000080:00200000:3.0:1788201707.067996:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c9b1bc54 fffffc0001029 2:0 ffff93c0c13bb9a0 118793 lru 00000080:00200000:3.0:1788201707.068000:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000036d49d37 118793: 1< 2 - - > 2< 486576128 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70378 - -> 00000080:00200000:3.0:1788201707.068001:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003e20841c 00000080:00200000:3.0:1788201707.068002:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.068003:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.068019:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.068028:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.068030:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.068031:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.068037:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.068039:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.068040:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.068043:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068048:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.068053:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.068054:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.068056:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.068058:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118793, 118793] 00000080:00200000:3.0:1788201707.068066:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.068067:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.068086:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.068088:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.068098:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486580224, count: 4096 00000080:00200000:3.0:1788201707.068104:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486580224, bytes: 4096 00000080:00200000:3.0:1788201707.068105:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.068108:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.068109:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486580224, 486584320) 0 0 00000080:00200000:3.0:1788201707.068110:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.068113:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118794, 118794] 00020000:00200000:3.0:1788201707.068115:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486580224 chunk: [486580224, 486584320) 486584320, 4096 00020000:00200000:3.0:1788201707.068116:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.068123:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118794, 118794] 00000008:00010000:3.0:1788201707.068126:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.068127:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.068132:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068138:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486580224 486584319) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068143:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068148:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068151:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.068155:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486580224, 486584320) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.068159:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118794 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.068162:0:17463:0:(rw26.c:898:ll_write_end()) pos 486580224, len 4096, copied 4096 00000080:00200000:3.0:1788201707.068165:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009e11debe[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118794] 00000080:00200000:3.0:1788201707.068166:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000dd24dcb6 fffffc0001029 2:0 ffff93c0c13bbb60 118794 lru 00000080:00200000:3.0:1788201707.068170:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000073e270f6 118794: 1< 2 - - > 2< 486580224 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70379 - -> 00000080:00200000:3.0:1788201707.068171:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009e11debe 00000080:00200000:3.0:1788201707.068172:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.068173:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.068179:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.068186:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.068188:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.068189:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.068195:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.068196:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.068198:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.068200:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068205:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.068209:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.068211:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.068213:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.068214:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118794, 118794] 00000080:00200000:3.0:1788201707.068222:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.068224:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.068241:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.068244:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.068253:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486584320, count: 4096 00000080:00200000:3.0:1788201707.068259:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486584320, bytes: 4096 00000080:00200000:3.0:1788201707.068261:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.068263:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.068264:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486584320, 486588416) 0 0 00000080:00200000:3.0:1788201707.068266:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.068269:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118795, 118795] 00020000:00200000:3.0:1788201707.068271:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486584320 chunk: [486584320, 486588416) 486588416, 4096 00020000:00200000:3.0:1788201707.068272:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.068279:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118795, 118795] 00000008:00010000:3.0:1788201707.068282:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.068283:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.068288:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068294:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486584320 486588415) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068299:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068303:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068307:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.068316:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486584320, 486588416) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.068319:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118795 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.068322:0:17463:0:(rw26.c:898:ll_write_end()) pos 486584320, len 4096, copied 4096 00000080:00200000:3.0:1788201707.068324:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006e6f3a3a[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118795] 00000080:00200000:3.0:1788201707.068326:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000c5e9e9c fffffc0001029 2:0 ffff93c0c13bbc40 118795 lru 00000080:00200000:3.0:1788201707.068330:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007d65c2aa 118795: 1< 2 - - > 2< 486584320 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70380 - -> 00000080:00200000:3.0:1788201707.068331:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006e6f3a3a 00000080:00200000:3.0:1788201707.068332:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.068333:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.068338:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.068345:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.068347:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.068348:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.068354:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.068356:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.068357:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.068360:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068364:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.068368:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.068370:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.068372:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.068374:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118795, 118795] 00000080:00200000:3.0:1788201707.068382:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.068383:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.068401:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.068403:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.068413:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486588416, count: 4096 00000080:00200000:3.0:1788201707.068419:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486588416, bytes: 4096 00000080:00200000:3.0:1788201707.068420:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.068422:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.068423:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486588416, 486592512) 0 0 00000080:00200000:3.0:1788201707.068425:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.068428:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118796, 118796] 00020000:00200000:3.0:1788201707.068430:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486588416 chunk: [486588416, 486592512) 486592512, 4096 00020000:00200000:3.0:1788201707.068431:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.068438:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118796, 118796] 00000008:00010000:3.0:1788201707.068440:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.068442:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.068447:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068452:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486588416 486592511) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068457:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068462:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068469:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.068474:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486588416, 486592512) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.068476:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118796 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.068480:0:17463:0:(rw26.c:898:ll_write_end()) pos 486588416, len 4096, copied 4096 00000080:00200000:3.0:1788201707.068482:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c40671ee[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118796] 00000080:00200000:3.0:1788201707.068484:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a3d6ced9 fffffc0001029 2:0 ffff93c0c13bcb60 118796 lru 00000080:00200000:3.0:1788201707.068487:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000003944958 118796: 1< 2 - - > 2< 486588416 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70381 - -> 00000080:00200000:3.0:1788201707.068488:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c40671ee 00000080:00200000:3.0:1788201707.068489:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.068490:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.068496:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.068503:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.068505:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.068506:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.068511:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.068513:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.068515:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.068517:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068522:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.068530:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.068531:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.068533:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.068535:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118796, 118796] 00000080:00200000:3.0:1788201707.068542:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.068544:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.068562:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.068564:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.068574:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486592512, count: 4096 00000080:00200000:3.0:1788201707.068580:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486592512, bytes: 4096 00000080:00200000:3.0:1788201707.068581:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.068583:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.068584:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486592512, 486596608) 0 0 00000080:00200000:3.0:1788201707.068586:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.068589:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118797, 118797] 00020000:00200000:3.0:1788201707.068591:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486592512 chunk: [486592512, 486596608) 486596608, 4096 00020000:00200000:3.0:1788201707.068593:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.068599:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118797, 118797] 00000008:00010000:3.0:1788201707.068602:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.068604:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.068608:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068614:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486592512 486596607) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068620:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068628:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068632:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.068636:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486592512, 486596608) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.068639:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118797 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.068642:0:17463:0:(rw26.c:898:ll_write_end()) pos 486592512, len 4096, copied 4096 00000080:00200000:3.0:1788201707.068644:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000569766f1[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118797] 00000080:00200000:3.0:1788201707.068646:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000275879e fffffc0001029 2:0 ffff93c0c13bcc40 118797 lru 00000080:00200000:3.0:1788201707.068649:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c1650506 118797: 1< 2 - - > 2< 486592512 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70382 - -> 00000080:00200000:3.0:1788201707.068650:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000569766f1 00000080:00200000:3.0:1788201707.068651:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.068652:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.068657:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.068665:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.068667:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.068668:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.068674:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.068675:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.068677:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.068679:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068687:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.068692:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.068693:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.068695:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.068697:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118797, 118797] 00000080:00200000:3.0:1788201707.068704:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.068706:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.068724:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.068726:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.068736:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486596608, count: 4096 00000080:00200000:3.0:1788201707.068742:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486596608, bytes: 4096 00000080:00200000:3.0:1788201707.068744:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.068746:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.068747:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486596608, 486600704) 0 0 00000080:00200000:3.0:1788201707.068749:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.068752:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118798, 118798] 00020000:00200000:3.0:1788201707.068754:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486596608 chunk: [486596608, 486600704) 486600704, 4096 00020000:00200000:3.0:1788201707.068755:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.068762:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118798, 118798] 00000008:00010000:3.0:1788201707.068765:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.068766:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.068771:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068776:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486596608 486600703) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068785:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068789:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068793:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.068798:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486596608, 486600704) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.068801:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118798 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.068804:0:17463:0:(rw26.c:898:ll_write_end()) pos 486596608, len 4096, copied 4096 00000080:00200000:3.0:1788201707.068807:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b6eaacfc[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118798] 00000080:00200000:3.0:1788201707.068808:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f0e2e760 fffffc0001029 2:0 ffff93c0c13bc540 118798 lru 00000080:00200000:3.0:1788201707.068812:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000aea6fc12 118798: 1< 2 - - > 2< 486596608 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70383 - -> 00000080:00200000:3.0:1788201707.068813:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b6eaacfc 00000080:00200000:3.0:1788201707.068814:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.068816:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.068821:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.068841:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.068843:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.068845:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.068850:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.068852:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.068854:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.068856:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068861:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.068866:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.068867:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.068870:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.068872:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118798, 118798] 00000080:00200000:3.0:1788201707.068879:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.068881:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.068898:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.068900:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.068910:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486600704, count: 4096 00000080:00200000:3.0:1788201707.068916:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486600704, bytes: 4096 00000080:00200000:3.0:1788201707.068918:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.068920:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.068921:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486600704, 486604800) 0 0 00000080:00200000:3.0:1788201707.068922:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.068926:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118799, 118799] 00020000:00200000:3.0:1788201707.068927:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486600704 chunk: [486600704, 486604800) 486604800, 4096 00020000:00200000:3.0:1788201707.068929:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.068935:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118799, 118799] 00000008:00010000:3.0:1788201707.068938:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.068940:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.068944:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068953:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486600704 486604799) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068959:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068963:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.068967:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.068971:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486600704, 486604800) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.068974:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118799 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.068978:0:17463:0:(rw26.c:898:ll_write_end()) pos 486600704, len 4096, copied 4096 00000080:00200000:3.0:1788201707.068980:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000eb313fab[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118799] 00000080:00200000:3.0:1788201707.068982:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ad44ae84 fffffc0001029 2:0 ffff93c0c13fbee0 118799 lru 00000080:00200000:3.0:1788201707.068985:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000110e8f44 118799: 1< 2 - - > 2< 486600704 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70384 - -> 00000080:00200000:3.0:1788201707.068987:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000eb313fab 00000080:00200000:3.0:1788201707.068988:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.068989:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.068994:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.069002:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.069004:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.069005:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.069023:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.069025:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.069027:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.069029:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069034:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.069039:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.069040:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.069043:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.069044:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118799, 118799] 00000080:00200000:3.0:1788201707.069052:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.069054:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.069072:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.069074:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.069085:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486604800, count: 4096 00000080:00200000:3.0:1788201707.069090:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486604800, bytes: 4096 00000080:00200000:3.0:1788201707.069092:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.069094:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.069095:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486604800, 486608896) 0 0 00000080:00200000:3.0:1788201707.069097:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.069100:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118800, 118800] 00020000:00200000:3.0:1788201707.069102:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486604800 chunk: [486604800, 486608896) 486608896, 4096 00020000:00200000:3.0:1788201707.069103:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.069110:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118800, 118800] 00000008:00010000:3.0:1788201707.069113:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.069115:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.069119:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069125:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486604800 486608895) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069131:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069135:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069139:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.069144:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486604800, 486608896) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.069147:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118800 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.069150:0:17463:0:(rw26.c:898:ll_write_end()) pos 486604800, len 4096, copied 4096 00000080:00200000:3.0:1788201707.069153:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ca48dd15[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118800] 00000080:00200000:3.0:1788201707.069154:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d725e9c4 fffffc0001029 2:0 ffff93c0c13fb7e0 118800 lru 00000080:00200000:3.0:1788201707.069158:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000042bd20d1 118800: 1< 2 - - > 2< 486604800 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70385 - -> 00000080:00200000:3.0:1788201707.069159:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ca48dd15 00000080:00200000:3.0:1788201707.069160:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.069161:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.069167:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.069174:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.069176:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.069178:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.069183:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.069185:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.069186:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.069189:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069193:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.069198:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.069199:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.069201:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.069203:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118800, 118800] 00000080:00200000:3.0:1788201707.069210:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.069212:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.069230:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.069232:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.069242:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486608896, count: 4096 00000080:00200000:3.0:1788201707.069248:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486608896, bytes: 4096 00000080:00200000:3.0:1788201707.069250:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.069252:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.069253:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486608896, 486612992) 0 0 00000080:00200000:3.0:1788201707.069255:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.069258:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118801, 118801] 00020000:00200000:3.0:1788201707.069259:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486608896 chunk: [486608896, 486612992) 486612992, 4096 00020000:00200000:3.0:1788201707.069261:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.069267:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118801, 118801] 00000008:00010000:3.0:1788201707.069271:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.069272:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.069277:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069282:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486608896 486612991) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069288:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069292:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069296:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.069300:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486608896, 486612992) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.069303:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118801 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.069306:0:17463:0:(rw26.c:898:ll_write_end()) pos 486608896, len 4096, copied 4096 00000080:00200000:3.0:1788201707.069308:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000077e51877[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118801] 00000080:00200000:3.0:1788201707.069310:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a75a07e6 fffffc0001029 2:0 ffff93c0c13fb2a0 118801 lru 00000080:00200000:3.0:1788201707.069314:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002e6adcb9 118801: 1< 2 - - > 2< 486608896 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70386 - -> 00000080:00200000:3.0:1788201707.069315:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000077e51877 00000080:00200000:3.0:1788201707.069316:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.069317:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.069322:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.069333:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.069334:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.069336:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.069341:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.069343:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.069344:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.069347:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069351:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.069355:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.069357:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.069359:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.069361:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118801, 118801] 00000080:00200000:3.0:1788201707.069369:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.069370:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.069388:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.069390:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.069400:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486612992, count: 4096 00000080:00200000:3.0:1788201707.069406:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486612992, bytes: 4096 00000080:00200000:3.0:1788201707.069408:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.069410:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.069411:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486612992, 486617088) 0 0 00000080:00200000:3.0:1788201707.069412:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.069416:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118802, 118802] 00020000:00200000:3.0:1788201707.069418:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486612992 chunk: [486612992, 486617088) 486617088, 4096 00020000:00200000:3.0:1788201707.069419:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.069426:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118802, 118802] 00000008:00010000:3.0:1788201707.069429:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.069430:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.069435:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069440:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486612992 486617087) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069446:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069450:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069454:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.069459:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486612992, 486617088) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.069462:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118802 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.069465:0:17463:0:(rw26.c:898:ll_write_end()) pos 486612992, len 4096, copied 4096 00000080:00200000:3.0:1788201707.069467:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001b46d229[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118802] 00000080:00200000:3.0:1788201707.069469:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000014e710f8 fffffc0001029 2:0 ffff93c0c13fbe00 118802 lru 00000080:00200000:3.0:1788201707.069473:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007fa302e3 118802: 1< 2 - - > 2< 486612992 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70387 - -> 00000080:00200000:3.0:1788201707.069474:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001b46d229 00000080:00200000:3.0:1788201707.069475:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.069476:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.069481:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.069489:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.069490:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.069491:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.069497:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.069499:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.069500:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.069503:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069507:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.069512:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.069513:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.069515:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.069517:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118802, 118802] 00000080:00200000:3.0:1788201707.069524:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.069526:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.069544:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.069546:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.069556:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486617088, count: 4096 00000080:00200000:3.0:1788201707.069562:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486617088, bytes: 4096 00000080:00200000:3.0:1788201707.069563:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.069566:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.069567:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486617088, 486621184) 0 0 00000080:00200000:3.0:1788201707.069568:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.069571:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118803, 118803] 00020000:00200000:3.0:1788201707.069573:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486617088 chunk: [486617088, 486621184) 486621184, 4096 00020000:00200000:3.0:1788201707.069575:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.069581:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118803, 118803] 00000008:00010000:3.0:1788201707.069584:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.069586:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.069590:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069596:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486617088 486621183) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069601:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069606:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069609:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.069613:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486617088, 486621184) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.069616:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118803 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.069619:0:17463:0:(rw26.c:898:ll_write_end()) pos 486617088, len 4096, copied 4096 00000080:00200000:3.0:1788201707.069622:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009f61583e[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118803] 00000080:00200000:3.0:1788201707.069623:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c22a17eb fffffc0001029 2:0 ffff93c0c13fc000 118803 lru 00000080:00200000:3.0:1788201707.069627:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b78cf4f9 118803: 1< 2 - - > 2< 486617088 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70388 - -> 00000080:00200000:3.0:1788201707.069629:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009f61583e 00000080:00200000:3.0:1788201707.069630:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.069631:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.069636:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.069644:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.069645:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.069647:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.069652:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.069654:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.069655:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.069658:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069662:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.069667:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.069669:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.069671:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.069673:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118803, 118803] 00000080:00200000:3.0:1788201707.069680:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.069682:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.069700:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.069702:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.069712:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486621184, count: 4096 00000080:00200000:3.0:1788201707.069717:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486621184, bytes: 4096 00000080:00200000:3.0:1788201707.069719:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.069722:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.069723:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486621184, 486625280) 0 0 00000080:00200000:3.0:1788201707.069724:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.069727:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118804, 118804] 00020000:00200000:3.0:1788201707.069729:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486621184 chunk: [486621184, 486625280) 486625280, 4096 00020000:00200000:3.0:1788201707.069730:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.069737:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118804, 118804] 00000008:00010000:3.0:1788201707.069740:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.069741:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.069746:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069751:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486621184 486625279) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069757:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069761:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069765:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.069770:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486621184, 486625280) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.069773:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118804 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.069776:0:17463:0:(rw26.c:898:ll_write_end()) pos 486621184, len 4096, copied 4096 00000080:00200000:3.0:1788201707.069778:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e5c04555[3 0000000075f4bf38 2 1 00000000e0b1c86e 0 0 118804] 00000080:00200000:3.0:1788201707.069780:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007a394e91 fffffc0001029 2:0 ffff93c0c13fca80 118804 lru 00000080:00200000:3.0:1788201707.069784:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000826a82ea 118804: 1< 2 - - > 2< 486621184 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70389 - -> 00000080:00200000:3.0:1788201707.069785:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e5c04555 00000080:00200000:3.0:1788201707.069786:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.069787:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.069792:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.069799:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.069801:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.069802:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.069808:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.069810:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.069811:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.069814:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.069819:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.069823:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e37b8 (ffff93c1d864f098/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.069825:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f098/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.069827:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.069845:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118804, 118804] 00000080:00200000:3.0:1788201707.069853:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.069855:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.069921:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.069928:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.069956:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486625280, count: 4096 00000080:00200000:2.0:1788201707.069976:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486625280, bytes: 4096 00000080:00200000:2.0:1788201707.069979:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.069984:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.069985:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486625280, 486629376) 0 0 00000080:00200000:2.0:1788201707.069988:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.069994:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118805, 118805] 00020000:00200000:2.0:1788201707.069998:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486625280 chunk: [486625280, 486629376) 486629376, 4096 00020000:00200000:2.0:1788201707.070001:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.070034:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118805, 118805] 00000008:00010000:2.0:1788201707.070041:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.070044:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.070053:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070066:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486625280 486629375) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070081:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070088:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070095:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.070104:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486625280, 486629376) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.070109:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118805 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.070115:0:17463:0:(rw26.c:898:ll_write_end()) pos 486625280, len 4096, copied 4096 00000080:00200000:2.0:1788201707.070119:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000cfc0d7c5[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118805] 00000080:00200000:2.0:1788201707.070122:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b11aa4a5 fffffc0001029 2:0 ffff93c0c13fc380 118805 lru 00000080:00200000:2.0:1788201707.070128:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ef6ece60 118805: 1< 2 - - > 2< 486625280 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70390 - -> 00000080:00200000:2.0:1788201707.070130:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000cfc0d7c5 00000080:00200000:2.0:1788201707.070132:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.070134:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.070144:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.070158:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.070161:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.070163:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.070176:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.070179:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.070182:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.070187:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070195:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.070205:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.070208:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.070213:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.070217:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118805, 118805] 00000080:00200000:2.0:1788201707.070230:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.070233:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.070260:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.070264:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.070280:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486629376, count: 4096 00000080:00200000:2.0:1788201707.070291:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486629376, bytes: 4096 00000080:00200000:2.0:1788201707.070294:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.070298:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.070299:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486629376, 486633472) 0 0 00000080:00200000:2.0:1788201707.070302:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.070308:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118806, 118806] 00020000:00200000:2.0:1788201707.070311:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486629376 chunk: [486629376, 486633472) 486633472, 4096 00020000:00200000:2.0:1788201707.070313:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.070328:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118806, 118806] 00000008:00010000:2.0:1788201707.070334:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.070336:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.070345:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070361:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486629376 486633471) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070370:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070378:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070385:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.070393:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486629376, 486633472) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.070399:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118806 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.070404:0:17463:0:(rw26.c:898:ll_write_end()) pos 486629376, len 4096, copied 4096 00000080:00200000:2.0:1788201707.070408:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000eda29de0[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118806] 00000080:00200000:2.0:1788201707.070412:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006c479e39 fffffc0001029 2:0 ffff93c0c13fc8c0 118806 lru 00000080:00200000:2.0:1788201707.070418:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000050baba20 118806: 1< 2 - - > 2< 486629376 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70391 - -> 00000080:00200000:2.0:1788201707.070420:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000eda29de0 00000080:00200000:2.0:1788201707.070421:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.070423:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.070432:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.070446:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.070449:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.070452:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.070462:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.070465:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.070468:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.070472:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070480:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.070489:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.070492:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.070496:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.070500:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118806, 118806] 00000080:00200000:2.0:1788201707.070511:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.070515:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.070547:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.070551:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.070568:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486633472, count: 4096 00000080:00200000:2.0:1788201707.070579:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486633472, bytes: 4096 00000080:00200000:2.0:1788201707.070582:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.070587:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.070588:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486633472, 486637568) 0 0 00000080:00200000:2.0:1788201707.070591:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.070596:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118807, 118807] 00020000:00200000:2.0:1788201707.070599:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486633472 chunk: [486633472, 486637568) 486637568, 4096 00020000:00200000:2.0:1788201707.070602:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.070615:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118807, 118807] 00000008:00010000:2.0:1788201707.070622:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.070624:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.070633:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070642:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486633472 486637567) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070652:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070660:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070667:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.070675:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486633472, 486637568) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.070681:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118807 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.070686:0:17463:0:(rw26.c:898:ll_write_end()) pos 486633472, len 4096, copied 4096 00000080:00200000:2.0:1788201707.070690:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fb891deb[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118807] 00000080:00200000:2.0:1788201707.070693:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000083954789 fffffc0001029 2:0 ffff93c0c13fcee0 118807 lru 00000080:00200000:2.0:1788201707.070700:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005d617740 118807: 1< 2 - - > 2< 486633472 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70392 - -> 00000080:00200000:2.0:1788201707.070702:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fb891deb 00000080:00200000:2.0:1788201707.070703:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.070705:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.070713:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.070725:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.070728:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.070730:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.070740:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.070743:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.070745:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.070749:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070756:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.070764:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.070766:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.070777:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.070780:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118807, 118807] 00000080:00200000:2.0:1788201707.070790:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.070793:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.070822:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.070826:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.070858:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486637568, count: 4096 00000080:00200000:2.0:1788201707.070868:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486637568, bytes: 4096 00000080:00200000:2.0:1788201707.070871:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.070875:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.070876:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486637568, 486641664) 0 0 00000080:00200000:2.0:1788201707.070879:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.070883:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118808, 118808] 00020000:00200000:2.0:1788201707.070886:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486637568 chunk: [486637568, 486641664) 486641664, 4096 00020000:00200000:2.0:1788201707.070888:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.070899:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118808, 118808] 00000008:00010000:2.0:1788201707.070905:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.070907:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.070915:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070924:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486637568 486641663) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070933:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070940:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.070947:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.070955:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486637568, 486641664) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.070960:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118808 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.070965:0:17463:0:(rw26.c:898:ll_write_end()) pos 486637568, len 4096, copied 4096 00000080:00200000:2.0:1788201707.070969:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f99c0000[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118808] 00000080:00200000:2.0:1788201707.070973:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d0846091 fffffc0001029 2:0 ffff93c0c13fc7e0 118808 lru 00000080:00200000:2.0:1788201707.070979:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bce355dd 118808: 1< 2 - - > 2< 486637568 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70393 - -> 00000080:00200000:2.0:1788201707.070981:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f99c0000 00000080:00200000:2.0:1788201707.070983:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.070984:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.070992:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.071024:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.071027:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.071030:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.071039:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.071042:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.071044:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.071048:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071056:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.071065:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.071067:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.071071:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.071074:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118808, 118808] 00000080:00200000:2.0:1788201707.071085:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.071088:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.071115:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.071118:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.071133:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486641664, count: 4096 00000080:00200000:2.0:1788201707.071144:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486641664, bytes: 4096 00000080:00200000:2.0:1788201707.071147:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.071150:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.071152:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486641664, 486645760) 0 0 00000080:00200000:2.0:1788201707.071154:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.071159:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118809, 118809] 00020000:00200000:2.0:1788201707.071163:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486641664 chunk: [486641664, 486645760) 486645760, 4096 00020000:00200000:2.0:1788201707.071165:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.071176:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118809, 118809] 00000008:00010000:2.0:1788201707.071181:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.071183:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.071191:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071200:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486641664 486645759) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071209:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071217:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071225:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.071232:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486641664, 486645760) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.071237:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118809 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.071242:0:17463:0:(rw26.c:898:ll_write_end()) pos 486641664, len 4096, copied 4096 00000080:00200000:2.0:1788201707.071246:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003b783888[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118809] 00000080:00200000:2.0:1788201707.071250:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c4fb7f9e fffffc0001029 2:0 ffff93c0c13fc2a0 118809 lru 00000080:00200000:2.0:1788201707.071256:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c567e83a 118809: 1< 2 - - > 2< 486641664 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70394 - -> 00000080:00200000:2.0:1788201707.071258:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003b783888 00000080:00200000:2.0:1788201707.071260:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.071262:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.071270:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.071282:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.071285:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.071287:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.071296:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.071299:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.071301:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.071305:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071313:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.071321:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.071324:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.071328:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.071331:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118809, 118809] 00000080:00200000:2.0:1788201707.071341:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.071344:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.071371:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.071375:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.071390:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486645760, count: 4096 00000080:00200000:2.0:1788201707.071400:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486645760, bytes: 4096 00000080:00200000:2.0:1788201707.071402:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.071406:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.071408:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486645760, 486649856) 0 0 00000080:00200000:2.0:1788201707.071411:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.071416:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118810, 118810] 00020000:00200000:2.0:1788201707.071419:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486645760 chunk: [486645760, 486649856) 486649856, 4096 00020000:00200000:2.0:1788201707.071422:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.071434:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118810, 118810] 00000008:00010000:2.0:1788201707.071440:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.071442:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.071450:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071460:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486645760 486649855) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071469:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071477:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071484:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.071493:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486645760, 486649856) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.071498:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118810 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.071504:0:17463:0:(rw26.c:898:ll_write_end()) pos 486645760, len 4096, copied 4096 00000080:00200000:2.0:1788201707.071508:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fffc4dff[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118810] 00000080:00200000:2.0:1788201707.071511:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003ff03389 fffffc0001029 2:0 ffff93c0c13fce00 118810 lru 00000080:00200000:2.0:1788201707.071517:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000455420b5 118810: 1< 2 - - > 2< 486645760 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70395 - -> 00000080:00200000:2.0:1788201707.071522:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fffc4dff 00000080:00200000:2.0:1788201707.071523:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.071525:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.071534:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.071547:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.071550:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.071552:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.071561:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.071564:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.071567:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.071571:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071579:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.071588:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.071590:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.071595:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.071598:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118810, 118810] 00000080:00200000:2.0:1788201707.071609:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.071612:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.071642:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.071646:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.071663:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486649856, count: 4096 00000080:00200000:2.0:1788201707.071675:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486649856, bytes: 4096 00000080:00200000:2.0:1788201707.071679:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.071683:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.071684:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486649856, 486653952) 0 0 00000080:00200000:2.0:1788201707.071687:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.071692:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118811, 118811] 00020000:00200000:2.0:1788201707.071695:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486649856 chunk: [486649856, 486653952) 486653952, 4096 00020000:00200000:2.0:1788201707.071697:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.071709:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118811, 118811] 00000008:00010000:2.0:1788201707.071714:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.071717:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.071724:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071734:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486649856 486653951) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071743:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071751:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071758:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.071766:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486649856, 486653952) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.071771:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118811 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.071777:0:17463:0:(rw26.c:898:ll_write_end()) pos 486649856, len 4096, copied 4096 00000080:00200000:2.0:1788201707.071781:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d9841ee5[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118811] 00000080:00200000:2.0:1788201707.071785:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002d990636 fffffc0001029 2:0 ffff93c0c13fc1c0 118811 lru 00000080:00200000:2.0:1788201707.071790:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000097544138 118811: 1< 2 - - > 2< 486649856 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70396 - -> 00000080:00200000:2.0:1788201707.071792:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d9841ee5 00000080:00200000:2.0:1788201707.071793:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.071795:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.071805:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.071817:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.071820:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.071823:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.071849:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.071852:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.071855:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.071859:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.071868:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.071876:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.071879:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.071884:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.071887:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118811, 118811] 00000080:00200000:2.0:1788201707.071897:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.071900:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.071930:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.071934:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.071951:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486653952, count: 4096 00000080:00200000:2.0:1788201707.071962:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486653952, bytes: 4096 00000080:00200000:2.0:1788201707.071965:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.071969:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.071970:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486653952, 486658048) 0 0 00000080:00200000:2.0:1788201707.071973:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.071979:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118812, 118812] 00020000:00200000:2.0:1788201707.071982:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486653952 chunk: [486653952, 486658048) 486658048, 4096 00020000:00200000:2.0:1788201707.071984:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.071998:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118812, 118812] 00000008:00010000:2.0:1788201707.072003:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.072006:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.072029:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072039:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486653952 486658047) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072048:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072056:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072063:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.072071:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486653952, 486658048) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.072077:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118812 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.072082:0:17463:0:(rw26.c:898:ll_write_end()) pos 486653952, len 4096, copied 4096 00000080:00200000:2.0:1788201707.072087:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005ffb6f58[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118812] 00000080:00200000:2.0:1788201707.072090:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000acb0c42c fffffc0001029 2:0 ffff93c0c13fc620 118812 lru 00000080:00200000:2.0:1788201707.072096:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000df6d949b 118812: 1< 2 - - > 2< 486653952 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70397 - -> 00000080:00200000:2.0:1788201707.072098:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005ffb6f58 00000080:00200000:2.0:1788201707.072099:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.072101:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.072111:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.072124:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.072127:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.072130:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.072140:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.072143:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.072145:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.072150:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072158:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.072167:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.072170:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.072174:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.072177:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118812, 118812] 00000080:00200000:2.0:1788201707.072188:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.072191:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.072221:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.072226:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.072242:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486658048, count: 4096 00000080:00200000:2.0:1788201707.072254:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486658048, bytes: 4096 00000080:00200000:2.0:1788201707.072257:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.072261:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.072263:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486658048, 486662144) 0 0 00000080:00200000:2.0:1788201707.072265:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.072271:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118813, 118813] 00020000:00200000:2.0:1788201707.072274:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486658048 chunk: [486658048, 486662144) 486662144, 4096 00020000:00200000:2.0:1788201707.072276:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.072289:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118813, 118813] 00000008:00010000:2.0:1788201707.072294:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.072296:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.072304:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072313:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486658048 486662143) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072322:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072330:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072337:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.072345:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486658048, 486662144) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.072351:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118813 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.072356:0:17463:0:(rw26.c:898:ll_write_end()) pos 486658048, len 4096, copied 4096 00000080:00200000:2.0:1788201707.072360:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000668edcb4[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118813] 00000080:00200000:2.0:1788201707.072363:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cc571047 fffffc0001029 2:0 ffff93c0c13fcd20 118813 lru 00000080:00200000:2.0:1788201707.072369:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ac921593 118813: 1< 2 - - > 2< 486658048 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70398 - -> 00000080:00200000:2.0:1788201707.072372:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000668edcb4 00000080:00200000:2.0:1788201707.072373:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.072375:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.072383:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.072396:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.072398:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.072401:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.072410:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.072413:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.072415:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.072419:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072427:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.072436:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.072439:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.072444:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.072447:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118813, 118813] 00000080:00200000:2.0:1788201707.072457:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.072460:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.072492:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.072496:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.072513:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486662144, count: 4096 00000080:00200000:2.0:1788201707.072525:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486662144, bytes: 4096 00000080:00200000:2.0:1788201707.072528:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.072532:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.072534:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486662144, 486666240) 0 0 00000080:00200000:2.0:1788201707.072537:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.072542:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118814, 118814] 00020000:00200000:2.0:1788201707.072545:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486662144 chunk: [486662144, 486666240) 486666240, 4096 00020000:00200000:2.0:1788201707.072548:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.072560:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118814, 118814] 00000008:00010000:2.0:1788201707.072565:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.072567:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.072575:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072584:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486662144 486666239) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072593:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072601:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072609:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.072622:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486662144, 486666240) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.072627:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118814 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.072632:0:17463:0:(rw26.c:898:ll_write_end()) pos 486662144, len 4096, copied 4096 00000080:00200000:2.0:1788201707.072637:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003e768671[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118814] 00000080:00200000:2.0:1788201707.072640:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ef37951c fffffc0001029 2:0 ffff93c0c13fc9a0 118814 lru 00000080:00200000:2.0:1788201707.072646:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000efa4aa4d 118814: 1< 2 - - > 2< 486662144 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70399 - -> 00000080:00200000:2.0:1788201707.072648:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003e768671 00000080:00200000:2.0:1788201707.072649:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.072651:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.072660:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.072673:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.072676:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.072678:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.072688:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.072691:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.072694:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.072698:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072706:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.072715:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.072718:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.072722:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.072725:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118814, 118814] 00000080:00200000:2.0:1788201707.072736:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.072739:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.072767:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.072771:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.072787:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486666240, count: 4096 00000080:00200000:2.0:1788201707.072799:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486666240, bytes: 4096 00000080:00200000:2.0:1788201707.072802:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.072806:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.072808:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486666240, 486670336) 0 0 00000080:00200000:2.0:1788201707.072810:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.072815:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118815, 118815] 00020000:00200000:2.0:1788201707.072819:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486666240 chunk: [486666240, 486670336) 486670336, 4096 00020000:00200000:2.0:1788201707.072821:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.072852:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118815, 118815] 00000008:00010000:2.0:1788201707.072859:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.072861:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.072869:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072879:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486666240 486670335) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072888:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072896:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.072910:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.072918:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486666240, 486670336) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.072923:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118815 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.072928:0:17463:0:(rw26.c:898:ll_write_end()) pos 486666240, len 4096, copied 4096 00000080:00200000:2.0:1788201707.072932:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000039f9a70f[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118815] 00000080:00200000:2.0:1788201707.072935:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000028565a5f fffffc0001029 2:0 ffff93c0c13fcb60 118815 lru 00000080:00200000:2.0:1788201707.072941:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a109cc48 118815: 1< 2 - - > 2< 486666240 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70400 - -> 00000080:00200000:2.0:1788201707.072944:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000039f9a70f 00000080:00200000:2.0:1788201707.072945:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.072947:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.072955:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.072967:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.072970:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.072972:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.072983:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.072985:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.072988:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.072992:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073000:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.073029:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.073031:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.073037:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.073040:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118815, 118815] 00000080:00200000:2.0:1788201707.073051:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.073055:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.073087:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.073091:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.073107:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486670336, count: 4096 00000080:00200000:2.0:1788201707.073117:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486670336, bytes: 4096 00000080:00200000:2.0:1788201707.073120:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.073124:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.073126:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486670336, 486674432) 0 0 00000080:00200000:2.0:1788201707.073128:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.073133:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118816, 118816] 00020000:00200000:2.0:1788201707.073136:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486670336 chunk: [486670336, 486674432) 486674432, 4096 00020000:00200000:2.0:1788201707.073138:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.073152:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118816, 118816] 00000008:00010000:2.0:1788201707.073157:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.073160:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.073168:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073178:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486670336 486674431) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073187:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073202:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073210:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.073217:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486670336, 486674432) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.073222:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118816 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.073227:0:17463:0:(rw26.c:898:ll_write_end()) pos 486670336, len 4096, copied 4096 00000080:00200000:2.0:1788201707.073232:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003c1d0de7[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118816] 00000080:00200000:2.0:1788201707.073235:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002faa411b fffffc0001029 2:0 ffff93c0c13fcc40 118816 lru 00000080:00200000:2.0:1788201707.073241:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000df4a9a0c 118816: 1< 2 - - > 2< 486670336 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70401 - -> 00000080:00200000:2.0:1788201707.073243:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003c1d0de7 00000080:00200000:2.0:1788201707.073244:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.073246:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.073255:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.073268:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.073270:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.073272:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.073282:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.073285:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.073288:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.073292:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073306:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.073313:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.073316:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.073321:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.073324:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118816, 118816] 00000080:00200000:2.0:1788201707.073334:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.073338:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.073368:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.073372:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.073389:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486674432, count: 4096 00000080:00200000:2.0:1788201707.073400:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486674432, bytes: 4096 00000080:00200000:2.0:1788201707.073403:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.073407:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.073409:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486674432, 486678528) 0 0 00000080:00200000:2.0:1788201707.073411:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.073417:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118817, 118817] 00020000:00200000:2.0:1788201707.073420:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486674432 chunk: [486674432, 486678528) 486678528, 4096 00020000:00200000:2.0:1788201707.073422:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.073434:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118817, 118817] 00000008:00010000:2.0:1788201707.073439:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.073441:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.073449:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073458:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486674432 486678527) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073473:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073481:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073488:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.073496:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486674432, 486678528) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.073502:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118817 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.073507:0:17463:0:(rw26.c:898:ll_write_end()) pos 486674432, len 4096, copied 4096 00000080:00200000:2.0:1788201707.073512:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f9f133e9[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118817] 00000080:00200000:2.0:1788201707.073515:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d141081b fffffc0001029 2:0 ffff93c0c13fc540 118817 lru 00000080:00200000:2.0:1788201707.073521:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000671a9ad8 118817: 1< 2 - - > 2< 486674432 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70402 - -> 00000080:00200000:2.0:1788201707.073523:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f9f133e9 00000080:00200000:2.0:1788201707.073524:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.073528:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.073537:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.073551:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.073555:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.073557:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.073567:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.073570:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.073573:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.073577:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073585:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.073593:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.073595:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.073600:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.073603:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118817, 118817] 00000080:00200000:2.0:1788201707.073614:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.073617:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.073648:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.073652:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.073668:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486678528, count: 4096 00000080:00200000:2.0:1788201707.073680:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486678528, bytes: 4096 00000080:00200000:2.0:1788201707.073683:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.073687:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.073689:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486678528, 486682624) 0 0 00000080:00200000:2.0:1788201707.073691:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.073697:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118818, 118818] 00020000:00200000:2.0:1788201707.073700:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486678528 chunk: [486678528, 486682624) 486682624, 4096 00020000:00200000:2.0:1788201707.073702:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.073715:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118818, 118818] 00000008:00010000:2.0:1788201707.073721:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.073723:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.073731:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073746:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486678528 486682623) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073755:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073763:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073770:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.073777:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486678528, 486682624) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.073782:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118818 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.073787:0:17463:0:(rw26.c:898:ll_write_end()) pos 486678528, len 4096, copied 4096 00000080:00200000:2.0:1788201707.073791:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b406764a[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118818] 00000080:00200000:2.0:1788201707.073794:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000019783ad9 fffffc0001029 2:0 ffff93c0c13fc460 118818 lru 00000080:00200000:2.0:1788201707.073799:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cac3ef0f 118818: 1< 2 - - > 2< 486678528 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70403 - -> 00000080:00200000:2.0:1788201707.073802:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b406764a 00000080:00200000:2.0:1788201707.073803:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.073805:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.073813:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.073825:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.073828:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.073846:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.073855:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.073858:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.073860:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.073865:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.073873:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.073881:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.073884:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.073889:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.073892:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118818, 118818] 00000080:00200000:2.0:1788201707.073903:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.073907:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.073935:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.073938:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.073953:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486682624, count: 4096 00000080:00200000:2.0:1788201707.073963:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486682624, bytes: 4096 00000080:00200000:2.0:1788201707.073966:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.073970:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.073972:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486682624, 486686720) 0 0 00000080:00200000:2.0:1788201707.073974:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.073979:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118819, 118819] 00020000:00200000:2.0:1788201707.073982:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486682624 chunk: [486682624, 486686720) 486686720, 4096 00020000:00200000:2.0:1788201707.073985:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.073995:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118819, 118819] 00000008:00010000:2.0:1788201707.074000:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.074002:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.074024:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074033:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486682624 486686719) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074042:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074050:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074058:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.074066:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486682624, 486686720) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.074071:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118819 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.074076:0:17463:0:(rw26.c:898:ll_write_end()) pos 486682624, len 4096, copied 4096 00000080:00200000:2.0:1788201707.074081:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000067b4562c[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118819] 00000080:00200000:2.0:1788201707.074084:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009a80602b fffffc0001029 2:0 ffff93c0c13fc0e0 118819 lru 00000080:00200000:2.0:1788201707.074090:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e4b021fc 118819: 1< 2 - - > 2< 486682624 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70404 - -> 00000080:00200000:2.0:1788201707.074092:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000067b4562c 00000080:00200000:2.0:1788201707.074093:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.074095:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.074104:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.074116:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.074119:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.074121:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.074131:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.074133:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.074136:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.074140:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074148:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.074157:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.074160:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.074164:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.074168:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118819, 118819] 00000080:00200000:2.0:1788201707.074178:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.074181:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.074211:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.074216:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.074233:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486686720, count: 4096 00000080:00200000:2.0:1788201707.074244:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486686720, bytes: 4096 00000080:00200000:2.0:1788201707.074247:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.074252:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.074254:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486686720, 486690816) 0 0 00000080:00200000:2.0:1788201707.074256:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.074262:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118820, 118820] 00020000:00200000:2.0:1788201707.074265:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486686720 chunk: [486686720, 486690816) 486690816, 4096 00020000:00200000:2.0:1788201707.074267:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.074281:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118820, 118820] 00000008:00010000:2.0:1788201707.074286:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.074289:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.074297:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074306:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486686720 486690815) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074315:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074323:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074330:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.074338:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486686720, 486690816) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.074344:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118820 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.074349:0:17463:0:(rw26.c:898:ll_write_end()) pos 486686720, len 4096, copied 4096 00000080:00200000:2.0:1788201707.074354:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007a93bef6[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118820] 00000080:00200000:2.0:1788201707.074357:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000075a92131 fffffc0001029 2:0 ffff93c0c13fc700 118820 lru 00000080:00200000:2.0:1788201707.074363:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e3bc8962 118820: 1< 2 - - > 2< 486686720 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70405 - -> 00000080:00200000:2.0:1788201707.074365:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007a93bef6 00000080:00200000:2.0:1788201707.074367:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.074368:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.074377:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.074395:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.074399:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.074401:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.074412:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.074415:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.074417:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.074422:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074430:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.074439:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.074442:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.074446:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.074450:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118820, 118820] 00000080:00200000:2.0:1788201707.074461:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.074465:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.074495:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.074499:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.074514:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486690816, count: 4096 00000080:00200000:2.0:1788201707.074524:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486690816, bytes: 4096 00000080:00200000:2.0:1788201707.074528:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.074531:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.074533:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486690816, 486694912) 0 0 00000080:00200000:2.0:1788201707.074536:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.074541:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118821, 118821] 00020000:00200000:2.0:1788201707.074544:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486690816 chunk: [486690816, 486694912) 486694912, 4096 00020000:00200000:2.0:1788201707.074547:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.074559:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118821, 118821] 00000008:00010000:2.0:1788201707.074565:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.074567:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.074575:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074584:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486690816 486694911) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074594:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074602:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074609:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.074617:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486690816, 486694912) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.074623:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118821 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.074628:0:17463:0:(rw26.c:898:ll_write_end()) pos 486690816, len 4096, copied 4096 00000080:00200000:2.0:1788201707.074632:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000241caf40[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118821] 00000080:00200000:2.0:1788201707.074635:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008a474502 fffffc0001029 2:0 ffff93c0c13fd540 118821 lru 00000080:00200000:2.0:1788201707.074641:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006fcf27af 118821: 1< 2 - - > 2< 486690816 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70406 - -> 00000080:00200000:2.0:1788201707.074644:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000241caf40 00000080:00200000:2.0:1788201707.074645:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.074647:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.074655:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.074668:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.074671:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.074673:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.074682:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.074685:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.074687:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.074691:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074699:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.074708:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.074710:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.074715:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.074718:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118821, 118821] 00000080:00200000:2.0:1788201707.074729:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.074732:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.074762:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.074765:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.074780:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486694912, count: 4096 00000080:00200000:2.0:1788201707.074791:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486694912, bytes: 4096 00000080:00200000:2.0:1788201707.074794:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.074798:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.074799:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486694912, 486699008) 0 0 00000080:00200000:2.0:1788201707.074802:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.074807:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118822, 118822] 00020000:00200000:2.0:1788201707.074811:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486694912 chunk: [486694912, 486699008) 486699008, 4096 00020000:00200000:2.0:1788201707.074813:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.074825:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118822, 118822] 00000008:00010000:2.0:1788201707.074845:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.074848:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.074856:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074863:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486694912 486699007) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074869:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074873:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074877:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.074883:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486694912, 486699008) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.074886:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118822 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.074891:0:17463:0:(rw26.c:898:ll_write_end()) pos 486694912, len 4096, copied 4096 00000080:00200000:2.0:1788201707.074893:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004c5fcaea[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118822] 00000080:00200000:2.0:1788201707.074895:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c6f17b41 fffffc0001029 2:0 ffff93c0c13fd460 118822 lru 00000080:00200000:2.0:1788201707.074899:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000011c6712d 118822: 1< 2 - - > 2< 486694912 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70407 - -> 00000080:00200000:2.0:1788201707.074902:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004c5fcaea 00000080:00200000:2.0:1788201707.074902:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.074904:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.074910:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.074918:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.074920:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.074921:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.074928:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.074930:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.074931:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.074934:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.074939:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.074944:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.074945:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.074948:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.074950:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118822, 118822] 00000080:00200000:2.0:1788201707.074956:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.074958:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.074977:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.074979:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.074989:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486699008, count: 4096 00000080:00200000:2.0:1788201707.074995:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486699008, bytes: 4096 00000080:00200000:2.0:1788201707.074997:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.075000:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.075001:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486699008, 486703104) 0 0 00000080:00200000:2.0:1788201707.075003:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.075006:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118823, 118823] 00020000:00200000:2.0:1788201707.075021:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486699008 chunk: [486699008, 486703104) 486703104, 4096 00020000:00200000:2.0:1788201707.075024:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.075033:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118823, 118823] 00000008:00010000:2.0:1788201707.075036:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.075037:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.075043:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075049:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486699008 486703103) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075054:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075059:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075063:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.075067:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486699008, 486703104) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.075070:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118823 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.075073:0:17463:0:(rw26.c:898:ll_write_end()) pos 486699008, len 4096, copied 4096 00000080:00200000:2.0:1788201707.075076:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000096916d25[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118823] 00000080:00200000:2.0:1788201707.075078:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007d7a5c28 fffffc0001029 2:0 ffff93c0c13fd0e0 118823 lru 00000080:00200000:2.0:1788201707.075082:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b0c05836 118823: 1< 2 - - > 2< 486699008 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70408 - -> 00000080:00200000:2.0:1788201707.075084:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000096916d25 00000080:00200000:2.0:1788201707.075085:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.075086:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.075091:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.075099:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.075101:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.075102:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.075108:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.075110:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.075112:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.075114:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075119:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.075124:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.075125:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.075128:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.075130:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118823, 118823] 00000080:00200000:2.0:1788201707.075136:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.075138:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.075156:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.075158:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.075169:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486703104, count: 4096 00000080:00200000:2.0:1788201707.075175:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486703104, bytes: 4096 00000080:00200000:2.0:1788201707.075177:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.075179:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.075180:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486703104, 486707200) 0 0 00000080:00200000:2.0:1788201707.075182:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.075185:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118824, 118824] 00020000:00200000:2.0:1788201707.075187:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486703104 chunk: [486703104, 486707200) 486707200, 4096 00020000:00200000:2.0:1788201707.075188:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.075194:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118824, 118824] 00000008:00010000:2.0:1788201707.075198:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.075199:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.075204:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075210:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486703104 486707199) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075215:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075220:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075224:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.075228:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486703104, 486707200) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.075230:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118824 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.075234:0:17463:0:(rw26.c:898:ll_write_end()) pos 486703104, len 4096, copied 4096 00000080:00200000:2.0:1788201707.075237:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000002323946[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118824] 00000080:00200000:2.0:1788201707.075239:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008bcbf20b fffffc0001029 2:0 ffff93c0c13fd700 118824 lru 00000080:00200000:2.0:1788201707.075242:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000864f0215 118824: 1< 2 - - > 2< 486703104 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70409 - -> 00000080:00200000:2.0:1788201707.075244:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000002323946 00000080:00200000:2.0:1788201707.075245:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.075246:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.075251:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.075259:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.075261:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.075262:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.075268:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.075269:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.075271:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.075273:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075279:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.075283:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.075285:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.075287:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.075289:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118824, 118824] 00000080:00200000:2.0:1788201707.075295:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.075297:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.075315:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.075317:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.075327:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486707200, count: 4096 00000080:00200000:2.0:1788201707.075332:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486707200, bytes: 4096 00000080:00200000:2.0:1788201707.075334:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.075336:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.075337:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486707200, 486711296) 0 0 00000080:00200000:2.0:1788201707.075339:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.075342:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118825, 118825] 00020000:00200000:2.0:1788201707.075344:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486707200 chunk: [486707200, 486711296) 486711296, 4096 00020000:00200000:2.0:1788201707.075345:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.075351:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118825, 118825] 00000008:00010000:2.0:1788201707.075355:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.075356:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.075361:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075366:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486707200 486711295) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075372:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075376:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075380:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.075384:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486707200, 486711296) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.075387:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118825 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.075391:0:17463:0:(rw26.c:898:ll_write_end()) pos 486707200, len 4096, copied 4096 00000080:00200000:2.0:1788201707.075393:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000049670d99[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118825] 00000080:00200000:2.0:1788201707.075395:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006bbcd181 fffffc0001029 2:0 ffff93c0c13fd000 118825 lru 00000080:00200000:2.0:1788201707.075398:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000940bf84c 118825: 1< 2 - - > 2< 486707200 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70410 - -> 00000080:00200000:2.0:1788201707.075400:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000049670d99 00000080:00200000:2.0:1788201707.075401:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.075402:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.075407:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.075415:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.075416:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.075418:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.075423:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.075425:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.075426:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.075429:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075433:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.075437:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.075439:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.075442:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.075444:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118825, 118825] 00000080:00200000:2.0:1788201707.075450:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.075452:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.075470:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.075472:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.075482:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486711296, count: 4096 00000080:00200000:2.0:1788201707.075488:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486711296, bytes: 4096 00000080:00200000:2.0:1788201707.075489:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.075491:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.075492:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486711296, 486715392) 0 0 00000080:00200000:2.0:1788201707.075494:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.075497:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118826, 118826] 00020000:00200000:2.0:1788201707.075499:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486711296 chunk: [486711296, 486715392) 486715392, 4096 00020000:00200000:2.0:1788201707.075500:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.075506:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118826, 118826] 00000008:00010000:2.0:1788201707.075510:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.075511:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.075516:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075522:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486711296 486715391) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075527:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075532:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075535:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.075543:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486711296, 486715392) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.075546:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118826 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.075549:0:17463:0:(rw26.c:898:ll_write_end()) pos 486711296, len 4096, copied 4096 00000080:00200000:2.0:1788201707.075552:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000971b49e3[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118826] 00000080:00200000:2.0:1788201707.075554:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ac7bc0d4 fffffc0001029 2:0 ffff93c0c13fda80 118826 lru 00000080:00200000:2.0:1788201707.075557:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003c7bc8d2 118826: 1< 2 - - > 2< 486711296 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70411 - -> 00000080:00200000:2.0:1788201707.075559:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000971b49e3 00000080:00200000:2.0:1788201707.075559:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.075560:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.075565:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.075573:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.075574:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.075575:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.075581:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.075583:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.075585:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.075587:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075591:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.075596:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.075597:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.075600:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.075602:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118826, 118826] 00000080:00200000:2.0:1788201707.075608:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.075610:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.075627:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.075629:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.075639:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486715392, count: 4096 00000080:00200000:2.0:1788201707.075645:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486715392, bytes: 4096 00000080:00200000:2.0:1788201707.075647:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.075649:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.075650:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486715392, 486719488) 0 0 00000080:00200000:2.0:1788201707.075651:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.075654:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118827, 118827] 00020000:00200000:2.0:1788201707.075656:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486715392 chunk: [486715392, 486719488) 486719488, 4096 00020000:00200000:2.0:1788201707.075658:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.075664:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118827, 118827] 00000008:00010000:2.0:1788201707.075667:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.075668:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.075673:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075679:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486715392 486719487) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075684:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075688:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075696:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.075700:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486715392, 486719488) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.075703:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118827 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.075706:0:17463:0:(rw26.c:898:ll_write_end()) pos 486715392, len 4096, copied 4096 00000080:00200000:2.0:1788201707.075708:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fae587a0[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118827] 00000080:00200000:2.0:1788201707.075710:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f6517e23 fffffc0001029 2:0 ffff93c0c13fd380 118827 lru 00000080:00200000:2.0:1788201707.075714:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007a78f1b6 118827: 1< 2 - - > 2< 486715392 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70412 - -> 00000080:00200000:2.0:1788201707.075715:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fae587a0 00000080:00200000:2.0:1788201707.075716:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.075717:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.075722:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.075729:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.075731:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.075732:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.075738:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.075740:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.075741:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.075744:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075749:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.075757:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.075759:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.075761:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.075763:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118827, 118827] 00000080:00200000:2.0:1788201707.075769:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.075772:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.075791:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.075794:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.075809:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486719488, count: 4096 00000080:00200000:2.0:1788201707.075820:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486719488, bytes: 4096 00000080:00200000:2.0:1788201707.075823:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.075827:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.075842:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486719488, 486723584) 0 0 00000080:00200000:2.0:1788201707.075845:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.075850:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118828, 118828] 00020000:00200000:2.0:1788201707.075852:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486719488 chunk: [486719488, 486723584) 486723584, 4096 00020000:00200000:2.0:1788201707.075854:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.075863:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118828, 118828] 00000008:00010000:2.0:1788201707.075866:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.075868:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.075873:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075880:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486719488 486723583) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075888:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075900:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075907:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.075915:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486719488, 486723584) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.075920:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118828 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.075925:0:17463:0:(rw26.c:898:ll_write_end()) pos 486719488, len 4096, copied 4096 00000080:00200000:2.0:1788201707.075929:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005091d726[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118828] 00000080:00200000:2.0:1788201707.075932:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002de714af fffffc0001029 2:0 ffff93c0c13fd8c0 118828 lru 00000080:00200000:2.0:1788201707.075937:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000991274ab 118828: 1< 2 - - > 2< 486719488 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70413 - -> 00000080:00200000:2.0:1788201707.075939:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005091d726 00000080:00200000:2.0:1788201707.075940:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.075942:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.075950:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.075962:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.075965:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.075966:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.075976:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.075978:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.075981:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.075985:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.075999:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.076018:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.076021:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.076024:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.076026:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118828, 118828] 00000080:00200000:2.0:1788201707.076034:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.076036:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.076055:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.076057:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.076067:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486723584, count: 4096 00000080:00200000:2.0:1788201707.076074:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486723584, bytes: 4096 00000080:00200000:2.0:1788201707.076075:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.076078:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.076079:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486723584, 486727680) 0 0 00000080:00200000:2.0:1788201707.076080:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.076084:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118829, 118829] 00020000:00200000:2.0:1788201707.076086:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486723584 chunk: [486723584, 486727680) 486727680, 4096 00020000:00200000:2.0:1788201707.076087:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.076094:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118829, 118829] 00000008:00010000:2.0:1788201707.076097:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.076099:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.076104:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076110:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486723584 486727679) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076120:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076124:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076128:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.076132:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486723584, 486727680) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.076135:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118829 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.076139:0:17463:0:(rw26.c:898:ll_write_end()) pos 486723584, len 4096, copied 4096 00000080:00200000:2.0:1788201707.076142:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b283675b[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118829] 00000080:00200000:2.0:1788201707.076144:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008d5b7b33 fffffc0001029 2:0 ffff93c0c13fdee0 118829 lru 00000080:00200000:2.0:1788201707.076150:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d0a732e9 118829: 1< 2 - - > 2< 486723584 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70414 - -> 00000080:00200000:2.0:1788201707.076152:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b283675b 00000080:00200000:2.0:1788201707.076153:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.076155:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.076163:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.076176:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.076179:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.076181:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.076191:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.076194:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.076197:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.076201:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076209:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.076217:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.076220:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.076224:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.076227:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118829, 118829] 00000080:00200000:2.0:1788201707.076237:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.076241:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.076268:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.076272:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.076286:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486727680, count: 4096 00000080:00200000:2.0:1788201707.076296:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486727680, bytes: 4096 00000080:00200000:2.0:1788201707.076299:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.076302:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.076304:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486727680, 486731776) 0 0 00000080:00200000:2.0:1788201707.076307:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.076312:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118830, 118830] 00020000:00200000:2.0:1788201707.076315:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486727680 chunk: [486727680, 486731776) 486731776, 4096 00020000:00200000:2.0:1788201707.076317:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.076328:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118830, 118830] 00000008:00010000:2.0:1788201707.076334:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.076336:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.076343:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076357:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486727680 486731775) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076367:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076375:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076383:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.076391:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486727680, 486731776) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.076397:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118830 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.076401:0:17463:0:(rw26.c:898:ll_write_end()) pos 486727680, len 4096, copied 4096 00000080:00200000:2.0:1788201707.076405:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008faf10c3[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118830] 00000080:00200000:2.0:1788201707.076408:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008e9a3e3d fffffc0001029 2:0 ffff93c0c13fd7e0 118830 lru 00000080:00200000:2.0:1788201707.076414:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000fce75c97 118830: 1< 2 - - > 2< 486727680 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70415 - -> 00000080:00200000:2.0:1788201707.076416:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008faf10c3 00000080:00200000:2.0:1788201707.076417:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.076419:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.076428:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.076440:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.076443:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.076445:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.076454:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.076457:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.076459:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.076463:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076471:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.076480:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.076482:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.076487:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.076490:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118830, 118830] 00000080:00200000:2.0:1788201707.076501:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.076504:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.076534:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.076538:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.076555:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486731776, count: 4096 00000080:00200000:2.0:1788201707.076567:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486731776, bytes: 4096 00000080:00200000:2.0:1788201707.076570:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.076574:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.076576:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486731776, 486735872) 0 0 00000080:00200000:2.0:1788201707.076578:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.076583:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118831, 118831] 00020000:00200000:2.0:1788201707.076587:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486731776 chunk: [486731776, 486735872) 486735872, 4096 00020000:00200000:2.0:1788201707.076589:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.076601:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118831, 118831] 00000008:00010000:2.0:1788201707.076607:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.076609:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.076617:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076627:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486731776 486735871) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076636:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076644:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076652:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.076659:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486731776, 486735872) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.076665:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118831 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.076671:0:17463:0:(rw26.c:898:ll_write_end()) pos 486731776, len 4096, copied 4096 00000080:00200000:2.0:1788201707.076675:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c05e22d9[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118831] 00000080:00200000:2.0:1788201707.076678:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ee394de6 fffffc0001029 2:0 ffff93c0c13fd2a0 118831 lru 00000080:00200000:2.0:1788201707.076684:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c27598da 118831: 1< 2 - - > 2< 486731776 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70416 - -> 00000080:00200000:2.0:1788201707.076686:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c05e22d9 00000080:00200000:2.0:1788201707.076687:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.076689:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.076698:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.076710:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.076713:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.076715:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.076724:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.076726:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.076729:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.076733:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076741:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.076750:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.076752:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.076757:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.076759:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118831, 118831] 00000080:00200000:2.0:1788201707.076769:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.076772:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.076800:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.076804:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.076820:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486735872, count: 4096 00000080:00200000:2.0:1788201707.076845:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486735872, bytes: 4096 00000080:00200000:2.0:1788201707.076848:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.076852:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.076853:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486735872, 486739968) 0 0 00000080:00200000:2.0:1788201707.076856:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.076861:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118832, 118832] 00020000:00200000:2.0:1788201707.076864:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486735872 chunk: [486735872, 486739968) 486739968, 4096 00020000:00200000:2.0:1788201707.076866:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.076878:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118832, 118832] 00000008:00010000:2.0:1788201707.076883:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.076885:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.076893:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076902:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486735872 486739967) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076911:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076919:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.076926:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.076935:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486735872, 486739968) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.076940:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118832 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.076946:0:17463:0:(rw26.c:898:ll_write_end()) pos 486735872, len 4096, copied 4096 00000080:00200000:2.0:1788201707.076950:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000290bc1f8[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118832] 00000080:00200000:2.0:1788201707.076953:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001f85ab7a fffffc0001029 2:0 ffff93c0c13fde00 118832 lru 00000080:00200000:2.0:1788201707.076958:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003f8e0415 118832: 1< 2 - - > 2< 486735872 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70417 - -> 00000080:00200000:2.0:1788201707.076960:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000290bc1f8 00000080:00200000:2.0:1788201707.076962:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.076963:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.076972:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.076988:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.076991:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.076993:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.077003:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.077005:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.077022:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.077026:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077034:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.077042:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.077044:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.077049:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.077052:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118832, 118832] 00000080:00200000:2.0:1788201707.077062:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.077065:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.077094:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.077098:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.077113:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486739968, count: 4096 00000080:00200000:2.0:1788201707.077124:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486739968, bytes: 4096 00000080:00200000:2.0:1788201707.077127:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.077131:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.077132:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486739968, 486744064) 0 0 00000080:00200000:2.0:1788201707.077134:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.077139:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118833, 118833] 00020000:00200000:2.0:1788201707.077143:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486739968 chunk: [486739968, 486744064) 486744064, 4096 00020000:00200000:2.0:1788201707.077145:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.077156:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118833, 118833] 00000008:00010000:2.0:1788201707.077161:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.077164:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.077172:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077180:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486739968 486744063) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077189:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077197:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077204:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.077212:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486739968, 486744064) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.077217:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118833 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.077221:0:17463:0:(rw26.c:898:ll_write_end()) pos 486739968, len 4096, copied 4096 00000080:00200000:2.0:1788201707.077225:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003ca812fd[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118833] 00000080:00200000:2.0:1788201707.077228:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000350d6718 fffffc0001029 2:0 ffff93c0c13fd1c0 118833 lru 00000080:00200000:2.0:1788201707.077235:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003cdf7b11 118833: 1< 2 - - > 2< 486739968 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70418 - -> 00000080:00200000:2.0:1788201707.077237:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003ca812fd 00000080:00200000:2.0:1788201707.077238:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.077240:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.077248:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.077261:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.077264:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.077266:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.077276:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.077279:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.077281:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.077285:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077293:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.077301:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.077303:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.077308:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.077311:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118833, 118833] 00000080:00200000:2.0:1788201707.077321:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.077325:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.077353:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.077356:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.077371:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486744064, count: 4096 00000080:00200000:2.0:1788201707.077380:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486744064, bytes: 4096 00000080:00200000:2.0:1788201707.077383:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.077386:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.077388:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486744064, 486748160) 0 0 00000080:00200000:2.0:1788201707.077391:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.077396:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118834, 118834] 00020000:00200000:2.0:1788201707.077399:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486744064 chunk: [486744064, 486748160) 486748160, 4096 00020000:00200000:2.0:1788201707.077401:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.077411:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118834, 118834] 00000008:00010000:2.0:1788201707.077416:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.077418:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.077426:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077432:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486744064 486748159) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077437:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077442:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077446:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.077451:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486744064, 486748160) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.077455:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118834 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.077458:0:17463:0:(rw26.c:898:ll_write_end()) pos 486744064, len 4096, copied 4096 00000080:00200000:2.0:1788201707.077461:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000073192ddb[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118834] 00000080:00200000:2.0:1788201707.077463:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000058c51de9 fffffc0001029 2:0 ffff93c0c13fd620 118834 lru 00000080:00200000:2.0:1788201707.077467:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000005ea7a6a6 118834: 1< 2 - - > 2< 486744064 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70419 - -> 00000080:00200000:2.0:1788201707.077469:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000073192ddb 00000080:00200000:2.0:1788201707.077470:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.077471:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.077477:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.077485:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.077487:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.077488:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.077494:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.077496:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.077497:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.077500:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077505:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.077509:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.077511:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.077513:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.077515:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118834, 118834] 00000080:00200000:2.0:1788201707.077522:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.077524:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.077541:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.077543:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.077553:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486748160, count: 4096 00000080:00200000:2.0:1788201707.077559:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486748160, bytes: 4096 00000080:00200000:2.0:1788201707.077561:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.077564:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.077565:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486748160, 486752256) 0 0 00000080:00200000:2.0:1788201707.077566:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.077570:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118835, 118835] 00020000:00200000:2.0:1788201707.077572:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486748160 chunk: [486748160, 486752256) 486752256, 4096 00020000:00200000:2.0:1788201707.077573:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.077580:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118835, 118835] 00000008:00010000:2.0:1788201707.077583:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.077584:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.077589:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077595:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486748160 486752255) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077600:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077605:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077609:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.077613:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486748160, 486752256) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.077616:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118835 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.077619:0:17463:0:(rw26.c:898:ll_write_end()) pos 486748160, len 4096, copied 4096 00000080:00200000:2.0:1788201707.077621:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000c74c5bc[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118835] 00000080:00200000:2.0:1788201707.077624:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c04a30b7 fffffc0001029 2:0 ffff93c0c13fdd20 118835 lru 00000080:00200000:2.0:1788201707.077627:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000017e766f2 118835: 1< 2 - - > 2< 486748160 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70420 - -> 00000080:00200000:2.0:1788201707.077629:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000c74c5bc 00000080:00200000:2.0:1788201707.077630:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.077631:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.077636:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.077643:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.077645:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.077646:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.077653:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.077656:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.077658:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.077662:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077669:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.077678:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.077680:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.077684:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.077687:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118835, 118835] 00000080:00200000:2.0:1788201707.077698:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.077701:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.077730:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.077734:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.077750:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486752256, count: 4096 00000080:00200000:2.0:1788201707.077761:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486752256, bytes: 4096 00000080:00200000:2.0:1788201707.077764:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.077768:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.077769:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486752256, 486756352) 0 0 00000080:00200000:2.0:1788201707.077772:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.077777:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118836, 118836] 00020000:00200000:2.0:1788201707.077780:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486752256 chunk: [486752256, 486756352) 486756352, 4096 00020000:00200000:2.0:1788201707.077782:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.077794:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118836, 118836] 00000008:00010000:2.0:1788201707.077800:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.077802:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.077810:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077818:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486752256 486756351) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077828:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077851:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077858:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.077866:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486752256, 486756352) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.077872:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118836 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.077877:0:17463:0:(rw26.c:898:ll_write_end()) pos 486752256, len 4096, copied 4096 00000080:00200000:2.0:1788201707.077881:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000901fc9df[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118836] 00000080:00200000:2.0:1788201707.077883:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f0f57e78 fffffc0001029 2:0 ffff93c0c13fd9a0 118836 lru 00000080:00200000:2.0:1788201707.077888:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b66d9c0f 118836: 1< 2 - - > 2< 486752256 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70421 - -> 00000080:00200000:2.0:1788201707.077890:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000901fc9df 00000080:00200000:2.0:1788201707.077891:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.077892:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.077901:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.077912:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.077915:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.077917:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.077925:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.077928:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.077930:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.077933:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.077940:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.077947:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.077949:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.077953:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.077956:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118836, 118836] 00000080:00200000:2.0:1788201707.077965:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.077969:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.077998:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.078002:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.078030:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486756352, count: 4096 00000080:00200000:2.0:1788201707.078042:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486756352, bytes: 4096 00000080:00200000:2.0:1788201707.078044:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.078048:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.078050:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486756352, 486760448) 0 0 00000080:00200000:2.0:1788201707.078052:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.078057:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118837, 118837] 00020000:00200000:2.0:1788201707.078060:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486756352 chunk: [486756352, 486760448) 486760448, 4096 00020000:00200000:2.0:1788201707.078063:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.078073:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118837, 118837] 00000008:00010000:2.0:1788201707.078078:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.078080:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.078087:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078094:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486756352 486760447) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078100:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078105:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078109:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.078114:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486756352, 486760448) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.078117:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118837 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.078122:0:17463:0:(rw26.c:898:ll_write_end()) pos 486756352, len 4096, copied 4096 00000080:00200000:2.0:1788201707.078126:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000534c54d4[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118837] 00000080:00200000:2.0:1788201707.078129:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000248e8550 fffffc0001029 2:0 ffff93c0c13fdb60 118837 lru 00000080:00200000:2.0:1788201707.078135:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000849a155b 118837: 1< 2 - - > 2< 486756352 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70422 - -> 00000080:00200000:2.0:1788201707.078137:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000534c54d4 00000080:00200000:2.0:1788201707.078138:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.078140:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.078149:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.078161:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.078164:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.078166:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.078175:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.078178:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.078180:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.078184:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078192:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.078200:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.078202:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.078207:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.078210:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118837, 118837] 00000080:00200000:2.0:1788201707.078221:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.078224:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.078253:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.078257:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.078272:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486760448, count: 4096 00000080:00200000:2.0:1788201707.078283:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486760448, bytes: 4096 00000080:00200000:2.0:1788201707.078286:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.078290:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.078291:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486760448, 486764544) 0 0 00000080:00200000:2.0:1788201707.078294:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.078299:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118838, 118838] 00020000:00200000:2.0:1788201707.078302:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486760448 chunk: [486760448, 486764544) 486764544, 4096 00020000:00200000:2.0:1788201707.078304:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.078317:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118838, 118838] 00000008:00010000:2.0:1788201707.078322:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.078324:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.078332:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078341:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486760448 486764543) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078349:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078357:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078364:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.078377:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486760448, 486764544) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.078382:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118838 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.078387:0:17463:0:(rw26.c:898:ll_write_end()) pos 486760448, len 4096, copied 4096 00000080:00200000:2.0:1788201707.078392:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fc1a0ef6[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118838] 00000080:00200000:2.0:1788201707.078395:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c61c2d2c fffffc0001029 2:0 ffff93c0c13fdc40 118838 lru 00000080:00200000:2.0:1788201707.078400:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007069134b 118838: 1< 2 - - > 2< 486760448 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70423 - -> 00000080:00200000:2.0:1788201707.078403:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fc1a0ef6 00000080:00200000:2.0:1788201707.078404:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.078406:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.078415:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.078427:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.078431:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.078433:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.078443:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.078445:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.078448:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.078453:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078461:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.078469:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.078472:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.078476:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.078479:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118838, 118838] 00000080:00200000:2.0:1788201707.078489:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.078493:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.078523:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.078527:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.078542:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486764544, count: 4096 00000080:00200000:2.0:1788201707.078552:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486764544, bytes: 4096 00000080:00200000:2.0:1788201707.078555:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.078558:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.078560:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486764544, 486768640) 0 0 00000080:00200000:2.0:1788201707.078562:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.078567:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118839, 118839] 00020000:00200000:2.0:1788201707.078570:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486764544 chunk: [486764544, 486768640) 486768640, 4096 00020000:00200000:2.0:1788201707.078572:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.078583:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118839, 118839] 00000008:00010000:2.0:1788201707.078588:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.078590:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.078597:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078607:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486764544 486768639) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078616:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078624:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078637:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.078645:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486764544, 486768640) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.078649:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118839 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.078653:0:17463:0:(rw26.c:898:ll_write_end()) pos 486764544, len 4096, copied 4096 00000080:00200000:2.0:1788201707.078657:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b7d4d5e1[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118839] 00000080:00200000:2.0:1788201707.078659:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003c379fec fffffc0001029 2:0 ffff93c0c13fed20 118839 lru 00000080:00200000:2.0:1788201707.078665:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f22c6146 118839: 1< 2 - - > 2< 486764544 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70424 - -> 00000080:00200000:2.0:1788201707.078667:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b7d4d5e1 00000080:00200000:2.0:1788201707.078668:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.078669:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.078677:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.078689:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.078692:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.078694:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.078703:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.078706:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.078708:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.078713:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078721:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.078735:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.078738:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.078743:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.078745:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118839, 118839] 00000080:00200000:2.0:1788201707.078755:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.078758:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.078784:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.078787:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.078803:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486768640, count: 4096 00000080:00200000:2.0:1788201707.078813:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486768640, bytes: 4096 00000080:00200000:2.0:1788201707.078815:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.078819:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.078820:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486768640, 486772736) 0 0 00000080:00200000:2.0:1788201707.078822:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.078827:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118840, 118840] 00020000:00200000:2.0:1788201707.078845:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486768640 chunk: [486768640, 486772736) 486772736, 4096 00020000:00200000:2.0:1788201707.078847:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.078860:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118840, 118840] 00000008:00010000:2.0:1788201707.078866:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.078868:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.078875:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078884:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486768640 486772735) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078893:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078906:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.078913:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.078921:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486768640, 486772736) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.078926:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118840 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.078931:0:17463:0:(rw26.c:898:ll_write_end()) pos 486768640, len 4096, copied 4096 00000080:00200000:2.0:1788201707.078935:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ce7fb2e4[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118840] 00000080:00200000:2.0:1788201707.078938:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fa17f0a4 fffffc0001029 2:0 ffff93c0c13fe9a0 118840 lru 00000080:00200000:2.0:1788201707.078944:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000051b7295f 118840: 1< 2 - - > 2< 486768640 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70425 - -> 00000080:00200000:2.0:1788201707.078946:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ce7fb2e4 00000080:00200000:2.0:1788201707.078947:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.078949:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.078958:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.078970:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.078972:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.078974:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.078984:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.078987:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.078989:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.078993:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079005:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.079029:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.079031:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.079036:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.079039:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118840, 118840] 00000080:00200000:2.0:1788201707.079051:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.079054:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.079083:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.079087:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.079104:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486772736, count: 4096 00000080:00200000:2.0:1788201707.079114:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486772736, bytes: 4096 00000080:00200000:2.0:1788201707.079117:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.079121:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.079122:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486772736, 486776832) 0 0 00000080:00200000:2.0:1788201707.079125:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.079130:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118841, 118841] 00020000:00200000:2.0:1788201707.079133:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486772736 chunk: [486772736, 486776832) 486776832, 4096 00020000:00200000:2.0:1788201707.079136:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.079148:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118841, 118841] 00000008:00010000:2.0:1788201707.079154:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.079156:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.079164:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079173:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486772736 486776831) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079187:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079195:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079202:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.079209:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486772736, 486776832) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.079214:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118841 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.079219:0:17463:0:(rw26.c:898:ll_write_end()) pos 486772736, len 4096, copied 4096 00000080:00200000:2.0:1788201707.079223:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c68f49fa[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118841] 00000080:00200000:2.0:1788201707.079226:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003769d9a1 fffffc0001029 2:0 ffff93c0c13feb60 118841 lru 00000080:00200000:2.0:1788201707.079232:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000febb1ef5 118841: 1< 2 - - > 2< 486772736 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70426 - -> 00000080:00200000:2.0:1788201707.079234:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c68f49fa 00000080:00200000:2.0:1788201707.079235:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.079237:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.079245:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.079258:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.079261:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.079262:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.079272:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.079275:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.079278:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.079282:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079291:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.079299:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.079301:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.079306:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.079309:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118841, 118841] 00000080:00200000:2.0:1788201707.079320:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.079323:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.079352:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.079356:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.079373:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486776832, count: 4096 00000080:00200000:2.0:1788201707.079384:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486776832, bytes: 4096 00000080:00200000:2.0:1788201707.079387:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.079391:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.079393:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486776832, 486780928) 0 0 00000080:00200000:2.0:1788201707.079395:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.079400:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118842, 118842] 00020000:00200000:2.0:1788201707.079403:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486776832 chunk: [486776832, 486780928) 486780928, 4096 00020000:00200000:2.0:1788201707.079405:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.079418:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118842, 118842] 00000008:00010000:2.0:1788201707.079424:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.079426:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.079434:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079449:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486776832 486780927) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079458:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079466:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079473:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.079481:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486776832, 486780928) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.079486:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118842 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.079491:0:17463:0:(rw26.c:898:ll_write_end()) pos 486776832, len 4096, copied 4096 00000080:00200000:2.0:1788201707.079496:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a9c2a137[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118842] 00000080:00200000:2.0:1788201707.079499:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000473f35e7 fffffc0001029 2:0 ffff93c0c13fec40 118842 lru 00000080:00200000:2.0:1788201707.079505:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000158e185b 118842: 1< 2 - - > 2< 486776832 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70427 - -> 00000080:00200000:2.0:1788201707.079508:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a9c2a137 00000080:00200000:2.0:1788201707.079509:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.079511:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.079520:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.079533:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.079536:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.079539:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.079548:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.079551:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.079554:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.079558:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079566:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.079575:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.079577:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.079582:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.079585:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118842, 118842] 00000080:00200000:2.0:1788201707.079595:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.079599:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.079629:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.079633:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.079651:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486780928, count: 4096 00000080:00200000:2.0:1788201707.079662:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486780928, bytes: 4096 00000080:00200000:2.0:1788201707.079665:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.079668:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.079670:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486780928, 486785024) 0 0 00000080:00200000:2.0:1788201707.079672:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.079678:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118843, 118843] 00020000:00200000:2.0:1788201707.079681:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486780928 chunk: [486780928, 486785024) 486785024, 4096 00020000:00200000:2.0:1788201707.079683:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.079695:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118843, 118843] 00000008:00010000:2.0:1788201707.079700:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.079703:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.079711:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079720:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486780928 486785023) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079729:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079738:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079745:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.079754:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486780928, 486785024) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.079759:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118843 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.079763:0:17463:0:(rw26.c:898:ll_write_end()) pos 486780928, len 4096, copied 4096 00000080:00200000:2.0:1788201707.079767:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007e58dd33[3 0000000075f4bf38 2 1 00000000d3a96578 0 0 118843] 00000080:00200000:2.0:1788201707.079770:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001ca1e8ac fffffc0001029 2:0 ffff93c0c13fe540 118843 lru 00000080:00200000:2.0:1788201707.079776:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002d53a5eb 118843: 1< 2 - - > 2< 486780928 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70428 - -> 00000080:00200000:2.0:1788201707.079778:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007e58dd33 00000080:00200000:2.0:1788201707.079779:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.079781:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.079789:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.079802:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.079805:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.079808:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.079818:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.079820:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.079823:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.079827:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.079866:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.079875:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702c68 (ffff93c1d300a998/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.079878:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a998/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.079883:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.079886:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118843, 118843] 00000080:00200000:2.0:1788201707.079897:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.079900:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.079978:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.079984:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.080017:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486785024, count: 4096 00000080:00200000:3.0:1788201707.080035:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486785024, bytes: 4096 00000080:00200000:3.0:1788201707.080038:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.080042:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.080044:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486785024, 486789120) 0 0 00000080:00200000:3.0:1788201707.080047:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.080053:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118844, 118844] 00020000:00200000:3.0:1788201707.080056:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486785024 chunk: [486785024, 486789120) 486789120, 4096 00020000:00200000:3.0:1788201707.080059:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.080075:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118844, 118844] 00000008:00010000:3.0:1788201707.080081:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.080084:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.080092:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080100:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486785024 486789119) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080106:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080114:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080122:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.080130:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486785024, 486789120) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.080136:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118844 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.080142:0:17463:0:(rw26.c:898:ll_write_end()) pos 486785024, len 4096, copied 4096 00000080:00200000:3.0:1788201707.080147:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000248e0e2d[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118844] 00000080:00200000:3.0:1788201707.080150:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000021f376e6 fffffc0001029 2:0 ffff93c0c13fe460 118844 lru 00000080:00200000:3.0:1788201707.080156:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000068612da 118844: 1< 2 - - > 2< 486785024 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70429 - -> 00000080:00200000:3.0:1788201707.080158:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000248e0e2d 00000080:00200000:3.0:1788201707.080159:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.080161:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.080171:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.080185:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.080188:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.080191:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.080203:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.080207:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.080209:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.080213:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080221:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.080229:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.080232:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.080237:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.080240:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118844, 118844] 00000080:00200000:3.0:1788201707.080257:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.080260:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.080294:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.080298:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.080318:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486789120, count: 4096 00000080:00200000:3.0:1788201707.080330:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486789120, bytes: 4096 00000080:00200000:3.0:1788201707.080333:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.080338:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.080339:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486789120, 486793216) 0 0 00000080:00200000:3.0:1788201707.080342:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.080348:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118845, 118845] 00020000:00200000:3.0:1788201707.080351:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486789120 chunk: [486789120, 486793216) 486793216, 4096 00020000:00200000:3.0:1788201707.080354:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.080367:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118845, 118845] 00000008:00010000:3.0:1788201707.080373:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.080376:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.080383:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080393:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486789120 486793215) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080401:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080409:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080416:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.080424:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486789120, 486793216) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.080430:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118845 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.080435:0:17463:0:(rw26.c:898:ll_write_end()) pos 486789120, len 4096, copied 4096 00000080:00200000:3.0:1788201707.080440:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000087745364[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118845] 00000080:00200000:3.0:1788201707.080443:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000042a0cb6b fffffc0001029 2:0 ffff93c0c13fe0e0 118845 lru 00000080:00200000:3.0:1788201707.080450:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007e31ef4a 118845: 1< 2 - - > 2< 486789120 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70430 - -> 00000080:00200000:3.0:1788201707.080452:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000087745364 00000080:00200000:3.0:1788201707.080453:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.080455:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.080464:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.080476:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.080479:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.080481:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.080490:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.080492:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.080494:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.080499:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080507:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.080524:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.080529:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.080533:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.080536:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118845, 118845] 00000080:00200000:3.0:1788201707.080547:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.080550:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.080580:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.080584:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.080602:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486793216, count: 4096 00000080:00200000:3.0:1788201707.080614:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486793216, bytes: 4096 00000080:00200000:3.0:1788201707.080617:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.080621:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.080622:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486793216, 486797312) 0 0 00000080:00200000:3.0:1788201707.080625:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.080630:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118846, 118846] 00020000:00200000:3.0:1788201707.080633:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486793216 chunk: [486793216, 486797312) 486797312, 4096 00020000:00200000:3.0:1788201707.080635:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.080647:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118846, 118846] 00000008:00010000:3.0:1788201707.080653:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.080655:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.080664:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080673:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486793216 486797311) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080681:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080689:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080696:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.080710:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486793216, 486797312) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.080715:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118846 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.080721:0:17463:0:(rw26.c:898:ll_write_end()) pos 486793216, len 4096, copied 4096 00000080:00200000:3.0:1788201707.080725:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e0316474[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118846] 00000080:00200000:3.0:1788201707.080728:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005008d1eb fffffc0001029 2:0 ffff93c0c13fe700 118846 lru 00000080:00200000:3.0:1788201707.080734:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004ae6a118 118846: 1< 2 - - > 2< 486793216 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70431 - -> 00000080:00200000:3.0:1788201707.080736:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e0316474 00000080:00200000:3.0:1788201707.080737:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.080740:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.080749:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.080761:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.080764:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.080766:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.080776:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.080779:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.080782:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.080786:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080794:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.080802:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.080804:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.080809:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.080812:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118846, 118846] 00000080:00200000:3.0:1788201707.080824:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.080827:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.080871:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.080875:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.080890:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486797312, count: 4096 00000080:00200000:3.0:1788201707.080900:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486797312, bytes: 4096 00000080:00200000:3.0:1788201707.080902:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.080906:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.080908:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486797312, 486801408) 0 0 00000080:00200000:3.0:1788201707.080911:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.080915:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118847, 118847] 00020000:00200000:3.0:1788201707.080918:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486797312 chunk: [486797312, 486801408) 486801408, 4096 00020000:00200000:3.0:1788201707.080921:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.080931:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118847, 118847] 00000008:00010000:3.0:1788201707.080936:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.080938:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.080946:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080955:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486797312 486801407) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080965:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080973:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.080985:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.080992:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486797312, 486801408) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.080997:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118847 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.081002:0:17463:0:(rw26.c:898:ll_write_end()) pos 486797312, len 4096, copied 4096 00000080:00200000:3.0:1788201707.081006:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000010100f6f[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118847] 00000080:00200000:3.0:1788201707.081022:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002697e52e fffffc0001029 2:0 ffff93c0c13fe000 118847 lru 00000080:00200000:3.0:1788201707.081027:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000091d085b5 118847: 1< 2 - - > 2< 486797312 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70432 - -> 00000080:00200000:3.0:1788201707.081029:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000010100f6f 00000080:00200000:3.0:1788201707.081030:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.081033:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.081041:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.081054:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.081057:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.081059:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.081069:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.081072:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.081075:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.081078:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081086:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.081100:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.081102:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.081106:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.081110:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118847, 118847] 00000080:00200000:3.0:1788201707.081121:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.081125:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.081154:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.081157:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.081174:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486801408, count: 4096 00000080:00200000:3.0:1788201707.081185:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486801408, bytes: 4096 00000080:00200000:3.0:1788201707.081188:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.081192:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.081194:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486801408, 486805504) 0 0 00000080:00200000:3.0:1788201707.081196:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.081201:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118848, 118848] 00020000:00200000:3.0:1788201707.081204:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486801408 chunk: [486801408, 486805504) 486805504, 4096 00020000:00200000:3.0:1788201707.081207:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.081220:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118848, 118848] 00000008:00010000:3.0:1788201707.081225:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.081228:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.081235:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081245:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486801408 486805503) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081254:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081267:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081274:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.081281:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486801408, 486805504) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.081286:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118848 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.081291:0:17463:0:(rw26.c:898:ll_write_end()) pos 486801408, len 4096, copied 4096 00000080:00200000:3.0:1788201707.081294:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000136d86b1[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118848] 00000080:00200000:3.0:1788201707.081297:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000df45ae38 fffffc0001029 2:0 ffff93c0c13fea80 118848 lru 00000080:00200000:3.0:1788201707.081303:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000050b14372 118848: 1< 2 - - > 2< 486801408 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70433 - -> 00000080:00200000:3.0:1788201707.081305:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000136d86b1 00000080:00200000:3.0:1788201707.081306:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.081308:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.081316:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.081327:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.081330:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.081332:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.081341:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.081343:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.081346:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.081350:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081362:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.081370:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.081372:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.081376:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.081379:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118848, 118848] 00000080:00200000:3.0:1788201707.081389:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.081393:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.081418:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.081421:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.081436:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486805504, count: 4096 00000080:00200000:3.0:1788201707.081445:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486805504, bytes: 4096 00000080:00200000:3.0:1788201707.081448:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.081452:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.081453:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486805504, 486809600) 0 0 00000080:00200000:3.0:1788201707.081456:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.081460:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118849, 118849] 00020000:00200000:3.0:1788201707.081463:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486805504 chunk: [486805504, 486809600) 486809600, 4096 00020000:00200000:3.0:1788201707.081466:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.081476:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118849, 118849] 00000008:00010000:3.0:1788201707.081481:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.081483:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.081489:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081498:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486805504 486809599) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081513:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081520:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081527:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.081536:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486805504, 486809600) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.081541:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118849 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.081546:0:17463:0:(rw26.c:898:ll_write_end()) pos 486805504, len 4096, copied 4096 00000080:00200000:3.0:1788201707.081550:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000320f4162[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118849] 00000080:00200000:3.0:1788201707.081553:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001b63bfc1 fffffc0001029 2:0 ffff93c0c13fe380 118849 lru 00000080:00200000:3.0:1788201707.081560:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bd4209c3 118849: 1< 2 - - > 2< 486805504 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70434 - -> 00000080:00200000:3.0:1788201707.081562:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000320f4162 00000080:00200000:3.0:1788201707.081563:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.081565:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.081573:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.081585:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.081588:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.081590:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.081599:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.081602:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.081605:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.081609:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081616:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.081625:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.081628:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.081632:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.081635:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118849, 118849] 00000080:00200000:3.0:1788201707.081646:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.081649:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.081679:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.081683:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.081700:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486809600, count: 4096 00000080:00200000:3.0:1788201707.081710:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486809600, bytes: 4096 00000080:00200000:3.0:1788201707.081713:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.081717:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.081718:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486809600, 486813696) 0 0 00000080:00200000:3.0:1788201707.081721:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.081726:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118850, 118850] 00020000:00200000:3.0:1788201707.081729:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486809600 chunk: [486809600, 486813696) 486813696, 4096 00020000:00200000:3.0:1788201707.081731:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.081743:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118850, 118850] 00000008:00010000:3.0:1788201707.081749:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.081751:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.081759:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081774:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486809600 486813695) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081783:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081792:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081799:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.081805:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486809600, 486813696) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.081810:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118850 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.081814:0:17463:0:(rw26.c:898:ll_write_end()) pos 486809600, len 4096, copied 4096 00000080:00200000:3.0:1788201707.081818:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000b20d8ab[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118850] 00000080:00200000:3.0:1788201707.081821:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e92b0da9 fffffc0001029 2:0 ffff93c0c13fe8c0 118850 lru 00000080:00200000:3.0:1788201707.081827:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000070617960 118850: 1< 2 - - > 2< 486809600 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70435 - -> 00000080:00200000:3.0:1788201707.081844:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000b20d8ab 00000080:00200000:3.0:1788201707.081845:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.081847:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.081855:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.081867:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.081869:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.081872:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.081880:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.081883:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.081885:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.081889:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.081898:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.081905:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.081908:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.081911:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.081913:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118850, 118850] 00000080:00200000:3.0:1788201707.081925:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.081928:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.081956:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.081960:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.081977:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486813696, count: 4096 00000080:00200000:3.0:1788201707.081987:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486813696, bytes: 4096 00000080:00200000:3.0:1788201707.081990:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.081994:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.081996:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486813696, 486817792) 0 0 00000080:00200000:3.0:1788201707.081998:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.082003:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118851, 118851] 00020000:00200000:3.0:1788201707.082006:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486813696 chunk: [486813696, 486817792) 486817792, 4096 00020000:00200000:3.0:1788201707.082021:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.082035:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118851, 118851] 00000008:00010000:3.0:1788201707.082040:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.082042:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.082050:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082059:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486813696 486817791) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082068:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082075:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082083:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.082092:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486813696, 486817792) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.082097:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118851 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.082102:0:17463:0:(rw26.c:898:ll_write_end()) pos 486813696, len 4096, copied 4096 00000080:00200000:3.0:1788201707.082106:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000074be551e[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118851] 00000080:00200000:3.0:1788201707.082109:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b69daf9e fffffc0001029 2:0 ffff93c0c13feee0 118851 lru 00000080:00200000:3.0:1788201707.082115:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000513f72a3 118851: 1< 2 - - > 2< 486813696 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70436 - -> 00000080:00200000:3.0:1788201707.082118:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000074be551e 00000080:00200000:3.0:1788201707.082119:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.082120:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.082129:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.082142:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.082145:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.082148:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.082157:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.082160:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.082162:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.082166:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082174:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.082182:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.082185:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.082189:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.082191:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118851, 118851] 00000080:00200000:3.0:1788201707.082202:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.082206:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.082234:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.082238:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.082254:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486817792, count: 4096 00000080:00200000:3.0:1788201707.082264:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486817792, bytes: 4096 00000080:00200000:3.0:1788201707.082266:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.082270:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.082272:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486817792, 486821888) 0 0 00000080:00200000:3.0:1788201707.082274:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.082279:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118852, 118852] 00020000:00200000:3.0:1788201707.082282:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486817792 chunk: [486817792, 486821888) 486821888, 4096 00020000:00200000:3.0:1788201707.082284:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.082295:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118852, 118852] 00000008:00010000:3.0:1788201707.082301:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.082303:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.082311:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082320:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486817792 486821887) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082330:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082337:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082344:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.082352:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486817792, 486821888) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.082357:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118852 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.082363:0:17463:0:(rw26.c:898:ll_write_end()) pos 486817792, len 4096, copied 4096 00000080:00200000:3.0:1788201707.082367:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004703ca1f[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118852] 00000080:00200000:3.0:1788201707.082370:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e17f5c77 fffffc0001029 2:0 ffff93c0c13fe7e0 118852 lru 00000080:00200000:3.0:1788201707.082375:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000fccf0135 118852: 1< 2 - - > 2< 486817792 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70437 - -> 00000080:00200000:3.0:1788201707.082377:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004703ca1f 00000080:00200000:3.0:1788201707.082378:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.082380:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.082389:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.082406:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.082409:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.082411:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.082421:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.082423:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.082426:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.082430:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082438:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.082446:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.082449:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.082453:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.082456:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118852, 118852] 00000080:00200000:3.0:1788201707.082467:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.082471:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.082500:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.082504:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.082520:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486821888, count: 4096 00000080:00200000:3.0:1788201707.082530:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486821888, bytes: 4096 00000080:00200000:3.0:1788201707.082533:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.082537:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.082538:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486821888, 486825984) 0 0 00000080:00200000:3.0:1788201707.082540:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.082546:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118853, 118853] 00020000:00200000:3.0:1788201707.082549:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486821888 chunk: [486821888, 486825984) 486825984, 4096 00020000:00200000:3.0:1788201707.082551:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.082563:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118853, 118853] 00000008:00010000:3.0:1788201707.082568:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.082571:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.082578:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082588:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486821888 486825983) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082597:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082604:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082612:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.082620:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486821888, 486825984) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.082625:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118853 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.082630:0:17463:0:(rw26.c:898:ll_write_end()) pos 486821888, len 4096, copied 4096 00000080:00200000:3.0:1788201707.082634:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000273c80ea[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118853] 00000080:00200000:3.0:1788201707.082637:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000508e482e fffffc0001029 2:0 ffff93c0c13fe2a0 118853 lru 00000080:00200000:3.0:1788201707.082644:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000393d0d2c 118853: 1< 2 - - > 2< 486821888 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70438 - -> 00000080:00200000:3.0:1788201707.082646:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000273c80ea 00000080:00200000:3.0:1788201707.082647:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.082649:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.082657:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.082668:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.082671:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.082673:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.082681:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.082684:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.082686:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.082690:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082698:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.082706:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.082708:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.082712:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.082714:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118853, 118853] 00000080:00200000:3.0:1788201707.082725:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.082727:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.082752:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.082755:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.082770:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486825984, count: 4096 00000080:00200000:3.0:1788201707.082779:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486825984, bytes: 4096 00000080:00200000:3.0:1788201707.082782:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.082786:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.082787:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486825984, 486830080) 0 0 00000080:00200000:3.0:1788201707.082790:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.082795:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118854, 118854] 00020000:00200000:3.0:1788201707.082798:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486825984 chunk: [486825984, 486830080) 486830080, 4096 00020000:00200000:3.0:1788201707.082800:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.082813:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118854, 118854] 00000008:00010000:3.0:1788201707.082819:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.082821:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.082844:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082854:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486825984 486830079) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082863:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082871:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082878:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.082885:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486825984, 486830080) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.082892:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118854 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.082897:0:17463:0:(rw26.c:898:ll_write_end()) pos 486825984, len 4096, copied 4096 00000080:00200000:3.0:1788201707.082901:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c419f635[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118854] 00000080:00200000:3.0:1788201707.082904:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cb265a0e fffffc0001029 2:0 ffff93c0c13fee00 118854 lru 00000080:00200000:3.0:1788201707.082910:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000627039e8 118854: 1< 2 - - > 2< 486825984 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70439 - -> 00000080:00200000:3.0:1788201707.082914:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c419f635 00000080:00200000:3.0:1788201707.082915:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.082917:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.082925:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.082936:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.082939:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.082941:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.082950:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.082952:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.082955:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.082959:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.082966:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.082974:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.082977:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.082981:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.082984:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118854, 118854] 00000080:00200000:3.0:1788201707.082996:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.082999:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.083044:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.083048:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.083063:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486830080, count: 4096 00000080:00200000:3.0:1788201707.083073:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486830080, bytes: 4096 00000080:00200000:3.0:1788201707.083076:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.083080:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.083081:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486830080, 486834176) 0 0 00000080:00200000:3.0:1788201707.083084:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.083088:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118855, 118855] 00020000:00200000:3.0:1788201707.083091:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486830080 chunk: [486830080, 486834176) 486834176, 4096 00020000:00200000:3.0:1788201707.083094:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.083104:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118855, 118855] 00000008:00010000:3.0:1788201707.083109:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.083111:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.083118:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083127:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486830080 486834175) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083136:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083144:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083151:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.083158:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486830080, 486834176) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.083163:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118855 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.083167:0:17463:0:(rw26.c:898:ll_write_end()) pos 486830080, len 4096, copied 4096 00000080:00200000:3.0:1788201707.083171:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000dbcba7fb[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118855] 00000080:00200000:3.0:1788201707.083174:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008229734e fffffc0001029 2:0 ffff93c0c13fe1c0 118855 lru 00000080:00200000:3.0:1788201707.083180:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bf6ba49c 118855: 1< 2 - - > 2< 486830080 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70440 - -> 00000080:00200000:3.0:1788201707.083182:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000dbcba7fb 00000080:00200000:3.0:1788201707.083183:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.083185:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.083192:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.083203:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.083205:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.083207:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.083215:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.083218:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.083220:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.083224:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083232:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.083240:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.083242:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.083246:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.083248:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118855, 118855] 00000080:00200000:3.0:1788201707.083259:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.083262:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.083291:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.083294:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.083312:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486834176, count: 4096 00000080:00200000:3.0:1788201707.083323:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486834176, bytes: 4096 00000080:00200000:3.0:1788201707.083326:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.083329:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.083331:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486834176, 486838272) 0 0 00000080:00200000:3.0:1788201707.083333:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.083338:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118856, 118856] 00020000:00200000:3.0:1788201707.083342:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486834176 chunk: [486834176, 486838272) 486838272, 4096 00020000:00200000:3.0:1788201707.083344:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.083356:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118856, 118856] 00000008:00010000:3.0:1788201707.083361:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.083363:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.083371:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083379:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486834176 486838271) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083388:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083395:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083402:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.083411:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486834176, 486838272) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.083415:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118856 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.083421:0:17463:0:(rw26.c:898:ll_write_end()) pos 486834176, len 4096, copied 4096 00000080:00200000:3.0:1788201707.083425:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c33d43fa[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118856] 00000080:00200000:3.0:1788201707.083428:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000043e3a6e9 fffffc0001029 2:0 ffff93c0c13fe620 118856 lru 00000080:00200000:3.0:1788201707.083434:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007258ca9c 118856: 1< 2 - - > 2< 486834176 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70441 - -> 00000080:00200000:3.0:1788201707.083436:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c33d43fa 00000080:00200000:3.0:1788201707.083438:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.083439:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.083449:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.083459:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.083462:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.083464:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.083473:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.083475:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.083478:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.083482:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083490:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.083498:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.083501:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.083505:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.083508:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118856, 118856] 00000080:00200000:3.0:1788201707.083520:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.083523:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.083553:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.083556:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.083573:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486838272, count: 4096 00000080:00200000:3.0:1788201707.083584:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486838272, bytes: 4096 00000080:00200000:3.0:1788201707.083587:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.083590:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.083592:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486838272, 486842368) 0 0 00000080:00200000:3.0:1788201707.083594:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.083599:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118857, 118857] 00020000:00200000:3.0:1788201707.083602:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486838272 chunk: [486838272, 486842368) 486842368, 4096 00020000:00200000:3.0:1788201707.083604:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.083616:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118857, 118857] 00000008:00010000:3.0:1788201707.083621:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.083623:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.083631:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083640:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486838272 486842367) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083649:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083656:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083663:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.083670:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486838272, 486842368) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.083675:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118857 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.083679:0:17463:0:(rw26.c:898:ll_write_end()) pos 486838272, len 4096, copied 4096 00000080:00200000:3.0:1788201707.083683:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000021a44417[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118857] 00000080:00200000:3.0:1788201707.083686:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004f25a2ae fffffc0001029 2:0 ffff93c0c13ff700 118857 lru 00000080:00200000:3.0:1788201707.083693:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000078566f76 118857: 1< 2 - - > 2< 486838272 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70442 - -> 00000080:00200000:3.0:1788201707.083695:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000021a44417 00000080:00200000:3.0:1788201707.083696:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.083698:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.083706:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.083714:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.083716:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.083717:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.083724:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.083726:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.083728:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.083730:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083735:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.083740:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.083741:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.083743:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.083745:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118857, 118857] 00000080:00200000:3.0:1788201707.083753:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.083754:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.083778:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.083782:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.083798:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486842368, count: 4096 00000080:00200000:3.0:1788201707.083809:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486842368, bytes: 4096 00000080:00200000:3.0:1788201707.083812:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.083816:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.083818:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486842368, 486846464) 0 0 00000080:00200000:3.0:1788201707.083820:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.083825:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118858, 118858] 00020000:00200000:3.0:1788201707.083828:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486842368 chunk: [486842368, 486846464) 486846464, 4096 00020000:00200000:3.0:1788201707.083845:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.083859:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118858, 118858] 00000008:00010000:3.0:1788201707.083863:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.083865:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.083873:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083882:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486842368 486846463) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083891:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083898:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083905:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.083919:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486842368, 486846464) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.083924:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118858 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.083930:0:17463:0:(rw26.c:898:ll_write_end()) pos 486842368, len 4096, copied 4096 00000080:00200000:3.0:1788201707.083934:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b8293e33[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118858] 00000080:00200000:3.0:1788201707.083937:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001205f44a fffffc0001029 2:0 ffff93c0c13ff000 118858 lru 00000080:00200000:3.0:1788201707.083943:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f3e0ede2 118858: 1< 2 - - > 2< 486842368 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70443 - -> 00000080:00200000:3.0:1788201707.083946:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b8293e33 00000080:00200000:3.0:1788201707.083947:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.083949:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.083957:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.083968:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.083971:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.083973:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.083981:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.083984:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.083986:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.083991:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.083999:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.084018:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.084021:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.084025:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.084028:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118858, 118858] 00000080:00200000:3.0:1788201707.084039:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.084042:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.084070:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.084074:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.084091:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486846464, count: 4096 00000080:00200000:3.0:1788201707.084102:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486846464, bytes: 4096 00000080:00200000:3.0:1788201707.084105:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.084109:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.084111:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486846464, 486850560) 0 0 00000080:00200000:3.0:1788201707.084113:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.084118:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118859, 118859] 00020000:00200000:3.0:1788201707.084121:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486846464 chunk: [486846464, 486850560) 486850560, 4096 00020000:00200000:3.0:1788201707.084123:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.084134:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118859, 118859] 00000008:00010000:3.0:1788201707.084139:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.084141:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.084148:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084157:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486846464 486850559) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084166:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084174:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084187:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.084195:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486846464, 486850560) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.084200:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118859 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.084205:0:17463:0:(rw26.c:898:ll_write_end()) pos 486846464, len 4096, copied 4096 00000080:00200000:3.0:1788201707.084209:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000878b865a[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118859] 00000080:00200000:3.0:1788201707.084212:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b7364836 fffffc0001029 2:0 ffff93c0c13ffa80 118859 lru 00000080:00200000:3.0:1788201707.084217:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000082eb2056 118859: 1< 2 - - > 2< 486846464 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70444 - -> 00000080:00200000:3.0:1788201707.084219:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000878b865a 00000080:00200000:3.0:1788201707.084220:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.084222:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.084231:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.084242:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.084245:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.084247:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.084257:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.084259:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.084262:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.084266:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084274:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.084287:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.084289:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.084293:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.084296:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118859, 118859] 00000080:00200000:3.0:1788201707.084307:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.084310:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.084339:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.084343:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.084360:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486850560, count: 4096 00000080:00200000:3.0:1788201707.084371:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486850560, bytes: 4096 00000080:00200000:3.0:1788201707.084374:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.084377:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.084379:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486850560, 486854656) 0 0 00000080:00200000:3.0:1788201707.084382:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.084386:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118860, 118860] 00020000:00200000:3.0:1788201707.084389:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486850560 chunk: [486850560, 486854656) 486854656, 4096 00020000:00200000:3.0:1788201707.084391:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.084403:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118860, 118860] 00000008:00010000:3.0:1788201707.084408:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.084410:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.084418:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084427:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486850560 486854655) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084435:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084448:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084454:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.084462:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486850560, 486854656) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.084468:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118860 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.084473:0:17463:0:(rw26.c:898:ll_write_end()) pos 486850560, len 4096, copied 4096 00000080:00200000:3.0:1788201707.084477:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ff4e3a4d[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118860] 00000080:00200000:3.0:1788201707.084480:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007d64e384 fffffc0001029 2:0 ffff93c0c13ff380 118860 lru 00000080:00200000:3.0:1788201707.084485:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009acc2b07 118860: 1< 2 - - > 2< 486850560 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70445 - -> 00000080:00200000:3.0:1788201707.084487:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ff4e3a4d 00000080:00200000:3.0:1788201707.084488:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.084490:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.084499:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.084511:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.084513:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.084516:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.084525:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.084527:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.084530:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.084534:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084548:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.084556:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.084559:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.084562:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.084565:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118860, 118860] 00000080:00200000:3.0:1788201707.084575:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.084578:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.084603:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.084606:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.084622:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486854656, count: 4096 00000080:00200000:3.0:1788201707.084631:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486854656, bytes: 4096 00000080:00200000:3.0:1788201707.084633:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.084637:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.084638:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486854656, 486858752) 0 0 00000080:00200000:3.0:1788201707.084641:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.084645:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118861, 118861] 00020000:00200000:3.0:1788201707.084648:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486854656 chunk: [486854656, 486858752) 486858752, 4096 00020000:00200000:3.0:1788201707.084650:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.084661:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118861, 118861] 00000008:00010000:3.0:1788201707.084665:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.084667:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.084674:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084683:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486854656 486858751) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084698:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084706:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084712:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.084719:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486854656, 486858752) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.084724:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118861 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.084729:0:17463:0:(rw26.c:898:ll_write_end()) pos 486854656, len 4096, copied 4096 00000080:00200000:3.0:1788201707.084732:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000f90f51f[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118861] 00000080:00200000:3.0:1788201707.084734:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c156c10d fffffc0001029 2:0 ffff93c0c13ff8c0 118861 lru 00000080:00200000:3.0:1788201707.084738:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000026a1d7f8 118861: 1< 2 - - > 2< 486854656 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70446 - -> 00000080:00200000:3.0:1788201707.084739:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000f90f51f 00000080:00200000:3.0:1788201707.084740:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.084741:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.084747:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.084755:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.084757:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.084758:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.084764:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.084766:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.084768:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.084772:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084780:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.084787:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.084790:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.084794:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.084796:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118861, 118861] 00000080:00200000:3.0:1788201707.084807:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.084811:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.084867:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.084871:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.084891:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486858752, count: 4096 00000080:00200000:3.0:1788201707.084903:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486858752, bytes: 4096 00000080:00200000:3.0:1788201707.084905:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.084909:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.084910:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486858752, 486862848) 0 0 00000080:00200000:3.0:1788201707.084913:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.084918:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118862, 118862] 00020000:00200000:3.0:1788201707.084921:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486858752 chunk: [486858752, 486862848) 486862848, 4096 00020000:00200000:3.0:1788201707.084923:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.084933:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118862, 118862] 00000008:00010000:3.0:1788201707.084938:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.084940:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.084948:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084962:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486858752 486862847) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084971:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084979:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.084986:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.084992:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486858752, 486862848) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.084997:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118862 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.085002:0:17463:0:(rw26.c:898:ll_write_end()) pos 486858752, len 4096, copied 4096 00000080:00200000:3.0:1788201707.085005:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000291fd8d6[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118862] 00000080:00200000:3.0:1788201707.085020:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004bbac134 fffffc0001029 2:0 ffff93c0c13ffee0 118862 lru 00000080:00200000:3.0:1788201707.085026:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000fd9519c9 118862: 1< 2 - - > 2< 486858752 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70447 - -> 00000080:00200000:3.0:1788201707.085027:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000291fd8d6 00000080:00200000:3.0:1788201707.085028:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.085030:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.085036:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.085045:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.085046:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.085048:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.085055:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.085057:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.085059:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.085061:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085066:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.085072:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.085073:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.085075:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.085077:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118862, 118862] 00000080:00200000:3.0:1788201707.085084:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.085086:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.085105:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.085107:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.085118:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486862848, count: 4096 00000080:00200000:3.0:1788201707.085124:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486862848, bytes: 4096 00000080:00200000:3.0:1788201707.085126:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.085128:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.085129:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486862848, 486866944) 0 0 00000080:00200000:3.0:1788201707.085130:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.085134:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118863, 118863] 00020000:00200000:3.0:1788201707.085136:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486862848 chunk: [486862848, 486866944) 486866944, 4096 00020000:00200000:3.0:1788201707.085137:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.085144:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118863, 118863] 00000008:00010000:3.0:1788201707.085147:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.085148:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.085154:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085160:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486862848 486866943) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085165:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085172:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085179:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.085187:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486862848, 486866944) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.085192:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118863 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.085196:0:17463:0:(rw26.c:898:ll_write_end()) pos 486862848, len 4096, copied 4096 00000080:00200000:3.0:1788201707.085200:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008f47b4e1[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118863] 00000080:00200000:3.0:1788201707.085203:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007dc89e93 fffffc0001029 2:0 ffff93c0c13ff7e0 118863 lru 00000080:00200000:3.0:1788201707.085209:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a40547df 118863: 1< 2 - - > 2< 486862848 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70448 - -> 00000080:00200000:3.0:1788201707.085211:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008f47b4e1 00000080:00200000:3.0:1788201707.085212:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.085214:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.085223:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.085234:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.085238:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.085240:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.085249:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.085252:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.085254:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.085258:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085266:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.085275:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.085278:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.085281:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.085284:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118863, 118863] 00000080:00200000:3.0:1788201707.085295:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.085298:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.085327:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.085330:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.085348:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486866944, count: 4096 00000080:00200000:3.0:1788201707.085358:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486866944, bytes: 4096 00000080:00200000:3.0:1788201707.085362:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.085365:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.085366:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486866944, 486871040) 0 0 00000080:00200000:3.0:1788201707.085369:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.085374:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118864, 118864] 00020000:00200000:3.0:1788201707.085376:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486866944 chunk: [486866944, 486871040) 486871040, 4096 00020000:00200000:3.0:1788201707.085379:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.085391:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118864, 118864] 00000008:00010000:3.0:1788201707.085397:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.085399:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.085406:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085415:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486866944 486871039) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085424:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085430:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085437:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.085444:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486866944, 486871040) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.085449:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118864 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.085454:0:17463:0:(rw26.c:898:ll_write_end()) pos 486866944, len 4096, copied 4096 00000080:00200000:3.0:1788201707.085458:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c6c78b02[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118864] 00000080:00200000:3.0:1788201707.085461:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005e20a5d1 fffffc0001029 2:0 ffff93c0c13ff2a0 118864 lru 00000080:00200000:3.0:1788201707.085466:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000002102153 118864: 1< 2 - - > 2< 486866944 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70449 - -> 00000080:00200000:3.0:1788201707.085468:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c6c78b02 00000080:00200000:3.0:1788201707.085469:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.085471:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.085478:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.085494:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.085497:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.085499:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.085508:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.085511:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.085513:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.085517:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085524:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.085533:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.085535:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.085539:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.085542:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118864, 118864] 00000080:00200000:3.0:1788201707.085552:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.085555:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.085582:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.085585:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.085599:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486871040, count: 4096 00000080:00200000:3.0:1788201707.085608:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486871040, bytes: 4096 00000080:00200000:3.0:1788201707.085610:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.085614:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.085616:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486871040, 486875136) 0 0 00000080:00200000:3.0:1788201707.085618:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.085623:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118865, 118865] 00020000:00200000:3.0:1788201707.085627:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486871040 chunk: [486871040, 486875136) 486875136, 4096 00020000:00200000:3.0:1788201707.085629:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.085641:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118865, 118865] 00000008:00010000:3.0:1788201707.085646:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.085648:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.085656:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085665:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486871040 486875135) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085674:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085681:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085688:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.085696:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486871040, 486875136) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.085701:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118865 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.085706:0:17463:0:(rw26.c:898:ll_write_end()) pos 486871040, len 4096, copied 4096 00000080:00200000:3.0:1788201707.085710:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009fec208d[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118865] 00000080:00200000:3.0:1788201707.085713:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c03e4286 fffffc0001029 2:0 ffff93c0c13ffe00 118865 lru 00000080:00200000:3.0:1788201707.085719:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000012e70d05 118865: 1< 2 - - > 2< 486871040 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70450 - -> 00000080:00200000:3.0:1788201707.085721:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009fec208d 00000080:00200000:3.0:1788201707.085722:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.085724:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.085733:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.085745:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.085748:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.085750:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.085758:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.085761:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.085764:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.085769:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085776:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.085785:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.085788:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.085792:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.085795:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118865, 118865] 00000080:00200000:3.0:1788201707.085806:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.085809:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.085853:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.085857:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.085873:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486875136, count: 4096 00000080:00200000:3.0:1788201707.085884:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486875136, bytes: 4096 00000080:00200000:3.0:1788201707.085886:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.085889:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.085891:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486875136, 486879232) 0 0 00000080:00200000:3.0:1788201707.085893:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.085898:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118866, 118866] 00020000:00200000:3.0:1788201707.085901:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486875136 chunk: [486875136, 486879232) 486879232, 4096 00020000:00200000:3.0:1788201707.085903:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.085914:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118866, 118866] 00000008:00010000:3.0:1788201707.085919:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.085921:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.085929:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085938:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486875136 486879231) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085947:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085955:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.085962:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.085969:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486875136, 486879232) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.085974:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118866 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.085980:0:17463:0:(rw26.c:898:ll_write_end()) pos 486875136, len 4096, copied 4096 00000080:00200000:3.0:1788201707.085983:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000aab5400e[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118866] 00000080:00200000:3.0:1788201707.085986:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000059b92a94 fffffc0001029 2:0 ffff93c0c13ff1c0 118866 lru 00000080:00200000:3.0:1788201707.085991:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003a3cfa39 118866: 1< 2 - - > 2< 486875136 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70451 - -> 00000080:00200000:3.0:1788201707.085995:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000aab5400e 00000080:00200000:3.0:1788201707.085996:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.085997:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.086017:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.086031:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.086034:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.086036:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.086046:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.086048:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.086051:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.086055:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086062:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.086070:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.086072:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.086076:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.086079:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118866, 118866] 00000080:00200000:3.0:1788201707.086091:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.086094:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.086122:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.086125:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.086142:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486879232, count: 4096 00000080:00200000:3.0:1788201707.086154:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486879232, bytes: 4096 00000080:00200000:3.0:1788201707.086157:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.086161:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.086162:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486879232, 486883328) 0 0 00000080:00200000:3.0:1788201707.086164:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.086169:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118867, 118867] 00020000:00200000:3.0:1788201707.086172:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486879232 chunk: [486879232, 486883328) 486883328, 4096 00020000:00200000:3.0:1788201707.086175:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.086188:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118867, 118867] 00000008:00010000:3.0:1788201707.086193:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.086195:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.086202:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086211:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486879232 486883327) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086220:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086228:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086235:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.086243:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486879232, 486883328) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.086248:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118867 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.086254:0:17463:0:(rw26.c:898:ll_write_end()) pos 486879232, len 4096, copied 4096 00000080:00200000:3.0:1788201707.086258:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009b3c6d80[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118867] 00000080:00200000:3.0:1788201707.086261:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a8af7fbb fffffc0001029 2:0 ffff93c0c13ff620 118867 lru 00000080:00200000:3.0:1788201707.086266:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007aefa3bb 118867: 1< 2 - - > 2< 486879232 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70452 - -> 00000080:00200000:3.0:1788201707.086269:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009b3c6d80 00000080:00200000:3.0:1788201707.086270:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.086272:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.086280:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.086292:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.086295:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.086297:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.086305:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.086308:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.086310:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.086314:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086322:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.086330:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.086333:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.086336:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.086339:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118867, 118867] 00000080:00200000:3.0:1788201707.086350:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.086352:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.086379:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.086382:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.086398:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486883328, count: 4096 00000080:00200000:3.0:1788201707.086409:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486883328, bytes: 4096 00000080:00200000:3.0:1788201707.086411:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.086415:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.086416:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486883328, 486887424) 0 0 00000080:00200000:3.0:1788201707.086419:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.086424:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118868, 118868] 00020000:00200000:3.0:1788201707.086427:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486883328 chunk: [486883328, 486887424) 486887424, 4096 00020000:00200000:3.0:1788201707.086428:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.086440:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118868, 118868] 00000008:00010000:3.0:1788201707.086445:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.086447:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.086455:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086463:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486883328 486887423) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086471:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086478:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086485:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.086493:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486883328, 486887424) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.086498:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118868 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.086503:0:17463:0:(rw26.c:898:ll_write_end()) pos 486883328, len 4096, copied 4096 00000080:00200000:3.0:1788201707.086507:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c8818385[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118868] 00000080:00200000:3.0:1788201707.086510:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000076a0f64d fffffc0001029 2:0 ffff93c0c13ffd20 118868 lru 00000080:00200000:3.0:1788201707.086516:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000f7f113a 118868: 1< 2 - - > 2< 486883328 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70453 - -> 00000080:00200000:3.0:1788201707.086518:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c8818385 00000080:00200000:3.0:1788201707.086519:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.086521:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.086529:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.086540:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.086543:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.086545:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.086553:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.086555:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.086557:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.086561:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086569:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.086577:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.086580:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.086583:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.086586:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118868, 118868] 00000080:00200000:3.0:1788201707.086596:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.086599:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.086630:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.086634:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.086652:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486887424, count: 4096 00000080:00200000:3.0:1788201707.086663:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486887424, bytes: 4096 00000080:00200000:3.0:1788201707.086665:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.086669:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.086671:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486887424, 486891520) 0 0 00000080:00200000:3.0:1788201707.086673:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.086679:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118869, 118869] 00020000:00200000:3.0:1788201707.086682:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486887424 chunk: [486887424, 486891520) 486891520, 4096 00020000:00200000:3.0:1788201707.086684:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.086695:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118869, 118869] 00000008:00010000:3.0:1788201707.086700:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.086703:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.086710:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086719:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486887424 486891519) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086728:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086735:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086742:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.086749:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486887424, 486891520) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.086755:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118869 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.086760:0:17463:0:(rw26.c:898:ll_write_end()) pos 486887424, len 4096, copied 4096 00000080:00200000:3.0:1788201707.086763:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000775d02ba[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118869] 00000080:00200000:3.0:1788201707.086766:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009a5909a4 fffffc0001029 2:0 ffff93c0c13ff9a0 118869 lru 00000080:00200000:3.0:1788201707.086771:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000018658e5e 118869: 1< 2 - - > 2< 486887424 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70454 - -> 00000080:00200000:3.0:1788201707.086773:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000775d02ba 00000080:00200000:3.0:1788201707.086774:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.086776:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.086783:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.086795:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.086798:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.086799:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.086808:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.086811:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.086813:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.086817:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086824:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.086848:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.086850:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.086855:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.086857:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118869, 118869] 00000080:00200000:3.0:1788201707.086869:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.086872:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.086899:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.086903:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.086920:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486891520, count: 4096 00000080:00200000:3.0:1788201707.086930:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486891520, bytes: 4096 00000080:00200000:3.0:1788201707.086933:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.086937:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.086938:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486891520, 486895616) 0 0 00000080:00200000:3.0:1788201707.086941:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.086946:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118870, 118870] 00020000:00200000:3.0:1788201707.086949:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486891520 chunk: [486891520, 486895616) 486895616, 4096 00020000:00200000:3.0:1788201707.086951:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.086962:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118870, 118870] 00000008:00010000:3.0:1788201707.086967:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.086969:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.086977:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086985:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486891520 486895615) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.086994:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087002:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087027:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.087045:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486891520, 486895616) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.087050:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118870 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.087056:0:17463:0:(rw26.c:898:ll_write_end()) pos 486891520, len 4096, copied 4096 00000080:00200000:3.0:1788201707.087059:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000eba28e94[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118870] 00000080:00200000:3.0:1788201707.087062:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003f51960e fffffc0001029 2:0 ffff93c0c13ffb60 118870 lru 00000080:00200000:3.0:1788201707.087067:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a90af8ff 118870: 1< 2 - - > 2< 486891520 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70455 - -> 00000080:00200000:3.0:1788201707.087069:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000eba28e94 00000080:00200000:3.0:1788201707.087070:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.087072:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.087080:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.087092:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.087095:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.087097:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.087106:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.087109:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.087112:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.087116:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087123:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.087131:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.087133:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.087138:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.087141:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118870, 118870] 00000080:00200000:3.0:1788201707.087152:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.087155:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.087184:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.087188:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.087205:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486895616, count: 4096 00000080:00200000:3.0:1788201707.087216:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486895616, bytes: 4096 00000080:00200000:3.0:1788201707.087219:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.087223:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.087224:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486895616, 486899712) 0 0 00000080:00200000:3.0:1788201707.087227:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.087232:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118871, 118871] 00020000:00200000:3.0:1788201707.087235:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486895616 chunk: [486895616, 486899712) 486899712, 4096 00020000:00200000:3.0:1788201707.087237:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.087249:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118871, 118871] 00000008:00010000:3.0:1788201707.087254:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.087257:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.087264:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087273:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486895616 486899711) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087282:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087290:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087301:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.087310:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486895616, 486899712) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.087315:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118871 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.087320:0:17463:0:(rw26.c:898:ll_write_end()) pos 486895616, len 4096, copied 4096 00000080:00200000:3.0:1788201707.087325:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000042ff12c[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118871] 00000080:00200000:3.0:1788201707.087328:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000122e4b6c fffffc0001029 2:0 ffff93c0c13ffc40 118871 lru 00000080:00200000:3.0:1788201707.087334:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000000e96fc 118871: 1< 2 - - > 2< 486895616 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70456 - -> 00000080:00200000:3.0:1788201707.087336:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000042ff12c 00000080:00200000:3.0:1788201707.087337:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.087339:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.087348:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.087359:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.087362:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.087364:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.087373:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.087376:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.087378:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.087382:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087390:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.087403:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.087406:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.087409:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.087412:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118871, 118871] 00000080:00200000:3.0:1788201707.087423:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.087426:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.087452:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.087455:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.087471:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486899712, count: 4096 00000080:00200000:3.0:1788201707.087481:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486899712, bytes: 4096 00000080:00200000:3.0:1788201707.087484:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.087488:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.087490:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486899712, 486903808) 0 0 00000080:00200000:3.0:1788201707.087492:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.087497:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118872, 118872] 00020000:00200000:3.0:1788201707.087500:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486899712 chunk: [486899712, 486903808) 486903808, 4096 00020000:00200000:3.0:1788201707.087502:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.087515:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118872, 118872] 00000008:00010000:3.0:1788201707.087520:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.087522:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.087529:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087538:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486899712 486903807) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087547:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087560:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087567:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.087575:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486899712, 486903808) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.087580:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118872 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.087585:0:17463:0:(rw26.c:898:ll_write_end()) pos 486899712, len 4096, copied 4096 00000080:00200000:3.0:1788201707.087589:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e4a305f0[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118872] 00000080:00200000:3.0:1788201707.087592:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008808331b fffffc0001029 2:0 ffff93c0c13ff540 118872 lru 00000080:00200000:3.0:1788201707.087598:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003317ead7 118872: 1< 2 - - > 2< 486899712 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70457 - -> 00000080:00200000:3.0:1788201707.087600:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e4a305f0 00000080:00200000:3.0:1788201707.087601:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.087603:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.087611:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.087623:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.087625:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.087627:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.087635:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.087638:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.087640:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.087644:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087657:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.087665:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.087667:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.087671:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.087673:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118872, 118872] 00000080:00200000:3.0:1788201707.087683:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.087686:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.087721:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.087723:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.087735:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486903808, count: 4096 00000080:00200000:3.0:1788201707.087742:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486903808, bytes: 4096 00000080:00200000:3.0:1788201707.087744:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.087746:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.087747:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486903808, 486907904) 0 0 00000080:00200000:3.0:1788201707.087749:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.087752:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118873, 118873] 00020000:00200000:3.0:1788201707.087754:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486903808 chunk: [486903808, 486907904) 486907904, 4096 00020000:00200000:3.0:1788201707.087757:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.087767:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118873, 118873] 00000008:00010000:3.0:1788201707.087773:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.087775:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.087782:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087791:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486903808 486907903) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087805:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087813:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087820:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.087827:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486903808, 486907904) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.087852:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118873 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.087857:0:17463:0:(rw26.c:898:ll_write_end()) pos 486903808, len 4096, copied 4096 00000080:00200000:3.0:1788201707.087862:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002bca1c52[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118873] 00000080:00200000:3.0:1788201707.087865:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001b3d6655 fffffc0001029 2:0 ffff93c0c13ff460 118873 lru 00000080:00200000:3.0:1788201707.087871:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000072b971ae 118873: 1< 2 - - > 2< 486903808 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70458 - -> 00000080:00200000:3.0:1788201707.087874:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002bca1c52 00000080:00200000:3.0:1788201707.087875:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.087877:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.087885:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.087897:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.087900:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.087902:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.087912:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.087914:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.087917:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.087921:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.087929:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.087937:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.087940:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.087944:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.087947:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118873, 118873] 00000080:00200000:3.0:1788201707.087958:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.087962:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.087990:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.087993:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.088020:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486907904, count: 4096 00000080:00200000:3.0:1788201707.088030:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486907904, bytes: 4096 00000080:00200000:3.0:1788201707.088032:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.088036:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.088038:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486907904, 486912000) 0 0 00000080:00200000:3.0:1788201707.088040:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.088045:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118874, 118874] 00020000:00200000:3.0:1788201707.088048:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486907904 chunk: [486907904, 486912000) 486912000, 4096 00020000:00200000:3.0:1788201707.088050:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.088061:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118874, 118874] 00000008:00010000:3.0:1788201707.088066:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.088068:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.088075:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088090:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486907904 486911999) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088099:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088106:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088112:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.088119:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486907904, 486912000) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.088124:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118874 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.088129:0:17463:0:(rw26.c:898:ll_write_end()) pos 486907904, len 4096, copied 4096 00000080:00200000:3.0:1788201707.088133:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000608d60dc[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118874] 00000080:00200000:3.0:1788201707.088136:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001c1969a4 fffffc0001029 2:0 ffff93c0c13ff0e0 118874 lru 00000080:00200000:3.0:1788201707.088141:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b1b043b1 118874: 1< 2 - - > 2< 486907904 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70459 - -> 00000080:00200000:3.0:1788201707.088143:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000608d60dc 00000080:00200000:3.0:1788201707.088145:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.088147:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.088155:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.088167:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.088169:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.088172:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.088181:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.088183:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.088186:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.088190:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088198:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.088206:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.088208:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.088212:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.088215:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118874, 118874] 00000080:00200000:3.0:1788201707.088226:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.088230:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.088258:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.088262:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.088277:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486912000, count: 4096 00000080:00200000:3.0:1788201707.088287:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486912000, bytes: 4096 00000080:00200000:3.0:1788201707.088289:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.088293:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.088295:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486912000, 486916096) 0 0 00000080:00200000:3.0:1788201707.088297:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.088302:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118875, 118875] 00020000:00200000:3.0:1788201707.088304:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486912000 chunk: [486912000, 486916096) 486916096, 4096 00020000:00200000:3.0:1788201707.088307:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.088317:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118875, 118875] 00000008:00010000:3.0:1788201707.088321:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.088323:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.088331:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088340:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486912000 486916095) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088348:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088355:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088362:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.088371:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486912000, 486916096) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.088376:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118875 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.088381:0:17463:0:(rw26.c:898:ll_write_end()) pos 486912000, len 4096, copied 4096 00000080:00200000:3.0:1788201707.088385:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004c428e49[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118875] 00000080:00200000:3.0:1788201707.088387:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b9af5b75 fffffc0001029 2:0 ffff93c0c4400d20 118875 lru 00000080:00200000:3.0:1788201707.088393:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e3615138 118875: 1< 2 - - > 2< 486912000 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70460 - -> 00000080:00200000:3.0:1788201707.088396:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004c428e49 00000080:00200000:3.0:1788201707.088397:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.088399:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.088407:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.088418:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.088421:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.088424:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.088432:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.088434:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.088437:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.088441:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088449:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.088457:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.088459:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.088463:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.088466:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118875, 118875] 00000080:00200000:3.0:1788201707.088476:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.088479:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.088508:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.088512:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.088529:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486916096, count: 4096 00000080:00200000:3.0:1788201707.088540:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486916096, bytes: 4096 00000080:00200000:3.0:1788201707.088542:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.088546:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.088548:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486916096, 486920192) 0 0 00000080:00200000:3.0:1788201707.088550:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.088555:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118876, 118876] 00020000:00200000:3.0:1788201707.088558:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486916096 chunk: [486916096, 486920192) 486920192, 4096 00020000:00200000:3.0:1788201707.088560:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.088570:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118876, 118876] 00000008:00010000:3.0:1788201707.088575:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.088577:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.088585:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088593:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486916096 486920191) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088602:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088609:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088616:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.088624:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486916096, 486920192) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.088629:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118876 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.088635:0:17463:0:(rw26.c:898:ll_write_end()) pos 486916096, len 4096, copied 4096 00000080:00200000:3.0:1788201707.088639:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000044f174ae[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118876] 00000080:00200000:3.0:1788201707.088642:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b018a892 fffffc0001029 2:0 ffff93c0c44009a0 118876 lru 00000080:00200000:3.0:1788201707.088648:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000eb392b1e 118876: 1< 2 - - > 2< 486916096 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70461 - -> 00000080:00200000:3.0:1788201707.088650:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000044f174ae 00000080:00200000:3.0:1788201707.088651:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.088653:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.088661:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.088678:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.088681:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.088683:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.088692:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.088694:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.088697:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.088701:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088709:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.088717:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.088720:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.088724:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.088727:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118876, 118876] 00000080:00200000:3.0:1788201707.088738:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.088741:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.088771:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.088775:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.088792:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486920192, count: 4096 00000080:00200000:3.0:1788201707.088804:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486920192, bytes: 4096 00000080:00200000:3.0:1788201707.088807:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.088810:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.088812:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486920192, 486924288) 0 0 00000080:00200000:3.0:1788201707.088814:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.088820:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118877, 118877] 00020000:00200000:3.0:1788201707.088823:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486920192 chunk: [486920192, 486924288) 486924288, 4096 00020000:00200000:3.0:1788201707.088825:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.088855:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118877, 118877] 00000008:00010000:3.0:1788201707.088861:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.088864:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.088871:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088881:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486920192 486924287) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088889:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088896:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088903:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.088911:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486920192, 486924288) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.088916:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118877 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.088922:0:17463:0:(rw26.c:898:ll_write_end()) pos 486920192, len 4096, copied 4096 00000080:00200000:3.0:1788201707.088926:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000513d2355[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118877] 00000080:00200000:3.0:1788201707.088929:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003ec4dc76 fffffc0001029 2:0 ffff93c0c4400b60 118877 lru 00000080:00200000:3.0:1788201707.088935:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f30c0cc9 118877: 1< 2 - - > 2< 486920192 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70462 - -> 00000080:00200000:3.0:1788201707.088937:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000513d2355 00000080:00200000:3.0:1788201707.088938:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.088941:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.088949:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.088961:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.088963:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.088965:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.088974:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.088977:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.088979:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.088983:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.088990:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.088998:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.089000:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.089004:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.089017:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118877, 118877] 00000080:00200000:3.0:1788201707.089030:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.089033:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.089061:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.089065:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.089082:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486924288, count: 4096 00000080:00200000:3.0:1788201707.089093:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486924288, bytes: 4096 00000080:00200000:3.0:1788201707.089095:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.089099:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.089100:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486924288, 486928384) 0 0 00000080:00200000:3.0:1788201707.089103:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.089108:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118878, 118878] 00020000:00200000:3.0:1788201707.089111:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486924288 chunk: [486924288, 486928384) 486928384, 4096 00020000:00200000:3.0:1788201707.089113:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.089125:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118878, 118878] 00000008:00010000:3.0:1788201707.089131:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.089133:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.089141:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089150:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486924288 486928383) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089158:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089165:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089172:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.089180:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486924288, 486928384) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.089184:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118878 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.089190:0:17463:0:(rw26.c:898:ll_write_end()) pos 486924288, len 4096, copied 4096 00000080:00200000:3.0:1788201707.089194:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000779c6fc4[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118878] 00000080:00200000:3.0:1788201707.089197:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000004d40ddc fffffc0001029 2:0 ffff93c0c4400c40 118878 lru 00000080:00200000:3.0:1788201707.089202:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a0569b76 118878: 1< 2 - - > 2< 486924288 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70463 - -> 00000080:00200000:3.0:1788201707.089205:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000779c6fc4 00000080:00200000:3.0:1788201707.089206:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.089209:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.089217:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.089228:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.089231:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.089233:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.089243:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.089245:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.089248:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.089252:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089260:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.089268:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.089270:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.089274:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.089277:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118878, 118878] 00000080:00200000:3.0:1788201707.089289:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.089293:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.089322:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.089326:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.089343:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486928384, count: 4096 00000080:00200000:3.0:1788201707.089354:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486928384, bytes: 4096 00000080:00200000:3.0:1788201707.089357:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.089360:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.089362:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486928384, 486932480) 0 0 00000080:00200000:3.0:1788201707.089364:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.089369:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118879, 118879] 00020000:00200000:3.0:1788201707.089372:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486928384 chunk: [486928384, 486932480) 486932480, 4096 00020000:00200000:3.0:1788201707.089374:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.089386:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118879, 118879] 00000008:00010000:3.0:1788201707.089391:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.089393:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.089400:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089409:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486928384 486932479) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089417:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089425:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089432:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.089439:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486928384, 486932480) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.089444:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118879 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.089449:0:17463:0:(rw26.c:898:ll_write_end()) pos 486928384, len 4096, copied 4096 00000080:00200000:3.0:1788201707.089453:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d1f827b5[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118879] 00000080:00200000:3.0:1788201707.089456:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e2d95c4f fffffc0001029 2:0 ffff93c0c4400540 118879 lru 00000080:00200000:3.0:1788201707.089462:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000589ee05 118879: 1< 2 - - > 2< 486928384 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70464 - -> 00000080:00200000:3.0:1788201707.089464:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d1f827b5 00000080:00200000:3.0:1788201707.089465:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.089467:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.089475:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.089487:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.089490:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.089492:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.089501:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.089504:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.089506:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.089511:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089518:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.089527:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.089530:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.089534:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.089537:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118879, 118879] 00000080:00200000:3.0:1788201707.089548:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.089551:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.089577:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.089581:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.089596:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486932480, count: 4096 00000080:00200000:3.0:1788201707.089605:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486932480, bytes: 4096 00000080:00200000:3.0:1788201707.089608:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.089611:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.089613:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486932480, 486936576) 0 0 00000080:00200000:3.0:1788201707.089615:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.089620:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118880, 118880] 00020000:00200000:3.0:1788201707.089623:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486932480 chunk: [486932480, 486936576) 486936576, 4096 00020000:00200000:3.0:1788201707.089625:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.089636:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118880, 118880] 00000008:00010000:3.0:1788201707.089641:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.089643:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.089650:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089659:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486932480 486936575) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089668:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089676:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089683:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.089691:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486932480, 486936576) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.089696:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118880 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.089701:0:17463:0:(rw26.c:898:ll_write_end()) pos 486932480, len 4096, copied 4096 00000080:00200000:3.0:1788201707.089706:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000096d74acb[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118880] 00000080:00200000:3.0:1788201707.089709:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f80c6998 fffffc0001029 2:0 ffff93c0c4400460 118880 lru 00000080:00200000:3.0:1788201707.089715:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007e3f1037 118880: 1< 2 - - > 2< 486932480 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70465 - -> 00000080:00200000:3.0:1788201707.089717:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000096d74acb 00000080:00200000:3.0:1788201707.089718:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.089720:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.089727:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.089739:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.089741:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.089743:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.089751:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.089754:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.089756:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.089761:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089769:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.089777:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.089779:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.089783:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.089786:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118880, 118880] 00000080:00200000:3.0:1788201707.089797:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.089800:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.089844:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.089848:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.089864:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486936576, count: 4096 00000080:00200000:3.0:1788201707.089874:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486936576, bytes: 4096 00000080:00200000:3.0:1788201707.089876:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.089880:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.089882:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486936576, 486940672) 0 0 00000080:00200000:3.0:1788201707.089884:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.089889:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118881, 118881] 00020000:00200000:3.0:1788201707.089892:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486936576 chunk: [486936576, 486940672) 486940672, 4096 00020000:00200000:3.0:1788201707.089894:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.089904:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118881, 118881] 00000008:00010000:3.0:1788201707.089909:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.089911:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.089918:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089927:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486936576 486940671) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089936:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089944:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.089951:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.089959:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486936576, 486940672) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.089964:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118881 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.089969:0:17463:0:(rw26.c:898:ll_write_end()) pos 486936576, len 4096, copied 4096 00000080:00200000:3.0:1788201707.089972:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000004c281fb[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118881] 00000080:00200000:3.0:1788201707.089975:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009298821d fffffc0001029 2:0 ffff93c0c44000e0 118881 lru 00000080:00200000:3.0:1788201707.089981:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f8759c21 118881: 1< 2 - - > 2< 486936576 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70466 - -> 00000080:00200000:3.0:1788201707.089983:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000004c281fb 00000080:00200000:3.0:1788201707.089984:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.089986:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.089995:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.090019:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.090023:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.090026:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.090035:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.090038:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.090041:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.090045:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090053:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.090061:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.090063:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.090068:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.090071:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118881, 118881] 00000080:00200000:3.0:1788201707.090082:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.090085:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.090114:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.090118:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.090133:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486940672, count: 4096 00000080:00200000:3.0:1788201707.090143:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486940672, bytes: 4096 00000080:00200000:3.0:1788201707.090145:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.090149:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.090151:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486940672, 486944768) 0 0 00000080:00200000:3.0:1788201707.090153:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.090158:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118882, 118882] 00020000:00200000:3.0:1788201707.090161:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486940672 chunk: [486940672, 486944768) 486944768, 4096 00020000:00200000:3.0:1788201707.090163:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.090173:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118882, 118882] 00000008:00010000:3.0:1788201707.090177:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.090180:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.090187:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090195:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486940672 486944767) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090204:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090211:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090219:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.090232:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486940672, 486944768) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.090237:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118882 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.090241:0:17463:0:(rw26.c:898:ll_write_end()) pos 486940672, len 4096, copied 4096 00000080:00200000:3.0:1788201707.090245:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bbf7dece[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118882] 00000080:00200000:3.0:1788201707.090248:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000097016510 fffffc0001029 2:0 ffff93c0c4400700 118882 lru 00000080:00200000:3.0:1788201707.090254:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007777ab60 118882: 1< 2 - - > 2< 486940672 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70467 - -> 00000080:00200000:3.0:1788201707.090256:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bbf7dece 00000080:00200000:3.0:1788201707.090257:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.090259:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.090267:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.090278:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.090280:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.090282:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.090290:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.090293:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.090295:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.090299:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090307:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.090315:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.090317:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.090321:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.090324:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118882, 118882] 00000080:00200000:3.0:1788201707.090334:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.090337:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.090362:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.090365:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.090380:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486944768, count: 4096 00000080:00200000:3.0:1788201707.090392:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486944768, bytes: 4096 00000080:00200000:3.0:1788201707.090394:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.090398:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.090399:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486944768, 486948864) 0 0 00000080:00200000:3.0:1788201707.090402:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.090406:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118883, 118883] 00020000:00200000:3.0:1788201707.090409:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486944768 chunk: [486944768, 486948864) 486948864, 4096 00020000:00200000:3.0:1788201707.090412:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.090422:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118883, 118883] 00000008:00010000:3.0:1788201707.090426:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.090428:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.090435:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090444:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486944768 486948863) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090452:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090459:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090471:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.090479:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486944768, 486948864) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.090484:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118883 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.090489:0:17463:0:(rw26.c:898:ll_write_end()) pos 486944768, len 4096, copied 4096 00000080:00200000:3.0:1788201707.090493:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009e1137ce[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118883] 00000080:00200000:3.0:1788201707.090496:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004714daf3 fffffc0001029 2:0 ffff93c0c4400000 118883 lru 00000080:00200000:3.0:1788201707.090502:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002e09debd 118883: 1< 2 - - > 2< 486944768 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70468 - -> 00000080:00200000:3.0:1788201707.090504:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009e1137ce 00000080:00200000:3.0:1788201707.090505:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.090507:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.090516:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.090527:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.090530:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.090533:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.090541:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.090544:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.090546:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.090551:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090558:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.090572:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.090574:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.090578:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.090582:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118883, 118883] 00000080:00200000:3.0:1788201707.090592:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.090595:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.090624:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.090628:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.090644:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486948864, count: 4096 00000080:00200000:3.0:1788201707.090655:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486948864, bytes: 4096 00000080:00200000:3.0:1788201707.090658:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.090661:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.090663:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486948864, 486952960) 0 0 00000080:00200000:3.0:1788201707.090665:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.090670:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118884, 118884] 00020000:00200000:3.0:1788201707.090673:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486948864 chunk: [486948864, 486952960) 486952960, 4096 00020000:00200000:3.0:1788201707.090676:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.090687:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118884, 118884] 00000008:00010000:3.0:1788201707.090692:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.090694:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.090701:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090710:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486948864 486952959) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090719:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090732:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090740:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.090747:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486948864, 486952960) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.090751:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118884 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.090755:0:17463:0:(rw26.c:898:ll_write_end()) pos 486948864, len 4096, copied 4096 00000080:00200000:3.0:1788201707.090759:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003e484310[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118884] 00000080:00200000:3.0:1788201707.090762:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e25af37b fffffc0001029 2:0 ffff93c0c4400a80 118884 lru 00000080:00200000:3.0:1788201707.090767:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000df13cadb 118884: 1< 2 - - > 2< 486948864 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70469 - -> 00000080:00200000:3.0:1788201707.090769:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003e484310 00000080:00200000:3.0:1788201707.090770:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.090772:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.090779:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.090791:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.090793:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.090795:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.090803:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.090806:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.090808:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.090812:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.090825:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.090857:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.090859:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.090863:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.090866:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118884, 118884] 00000080:00200000:3.0:1788201707.090880:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.090883:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.090915:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.090919:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.090936:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486952960, count: 4096 00000080:00200000:3.0:1788201707.090947:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486952960, bytes: 4096 00000080:00200000:3.0:1788201707.090949:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.090953:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.090954:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486952960, 486957056) 0 0 00000080:00200000:3.0:1788201707.090957:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.090962:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118885, 118885] 00020000:00200000:3.0:1788201707.090965:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486952960 chunk: [486952960, 486957056) 486957056, 4096 00020000:00200000:3.0:1788201707.090967:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.090980:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118885, 118885] 00000008:00010000:3.0:1788201707.090985:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.090987:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.090995:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091004:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486952960 486957055) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091029:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091037:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091044:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.091051:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486952960, 486957056) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.091057:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118885 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.091062:0:17463:0:(rw26.c:898:ll_write_end()) pos 486952960, len 4096, copied 4096 00000080:00200000:3.0:1788201707.091066:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000f4f93c0[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118885] 00000080:00200000:3.0:1788201707.091069:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d3dd259f fffffc0001029 2:0 ffff93c0c4400380 118885 lru 00000080:00200000:3.0:1788201707.091074:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000055977363 118885: 1< 2 - - > 2< 486952960 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70470 - -> 00000080:00200000:3.0:1788201707.091077:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000f4f93c0 00000080:00200000:3.0:1788201707.091078:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.091080:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.091088:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.091100:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.091103:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.091105:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.091115:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.091118:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.091121:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.091125:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091133:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.091140:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.091143:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.091147:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.091150:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118885, 118885] 00000080:00200000:3.0:1788201707.091161:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.091164:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.091192:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.091196:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.091213:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486957056, count: 4096 00000080:00200000:3.0:1788201707.091223:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486957056, bytes: 4096 00000080:00200000:3.0:1788201707.091226:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.091230:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.091231:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486957056, 486961152) 0 0 00000080:00200000:3.0:1788201707.091234:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.091239:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118886, 118886] 00020000:00200000:3.0:1788201707.091242:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486957056 chunk: [486957056, 486961152) 486961152, 4096 00020000:00200000:3.0:1788201707.091244:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.091257:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118886, 118886] 00000008:00010000:3.0:1788201707.091262:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.091264:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.091272:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091286:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486957056 486961151) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091295:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091303:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091310:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.091318:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486957056, 486961152) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.091322:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118886 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.091328:0:17463:0:(rw26.c:898:ll_write_end()) pos 486957056, len 4096, copied 4096 00000080:00200000:3.0:1788201707.091332:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c07b1472[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118886] 00000080:00200000:3.0:1788201707.091334:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004ef7c82c fffffc0001029 2:0 ffff93c0c44008c0 118886 lru 00000080:00200000:3.0:1788201707.091339:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f60974cc 118886: 1< 2 - - > 2< 486957056 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70471 - -> 00000080:00200000:3.0:1788201707.091341:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c07b1472 00000080:00200000:3.0:1788201707.091342:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.091344:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.091352:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.091364:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.091367:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.091370:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.091380:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.091383:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.091385:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.091389:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091397:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.091405:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.091408:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.091412:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.091415:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118886, 118886] 00000080:00200000:3.0:1788201707.091427:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.091430:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.091458:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.091462:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.091479:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486961152, count: 4096 00000080:00200000:3.0:1788201707.091490:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486961152, bytes: 4096 00000080:00200000:3.0:1788201707.091493:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.091496:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.091498:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486961152, 486965248) 0 0 00000080:00200000:3.0:1788201707.091501:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.091506:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118887, 118887] 00020000:00200000:3.0:1788201707.091509:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486961152 chunk: [486961152, 486965248) 486965248, 4096 00020000:00200000:3.0:1788201707.091511:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.091523:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118887, 118887] 00000008:00010000:3.0:1788201707.091529:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.091531:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.091539:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091548:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486961152 486965247) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091557:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091565:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091572:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.091579:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486961152, 486965248) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.091584:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118887 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.091589:0:17463:0:(rw26.c:898:ll_write_end()) pos 486961152, len 4096, copied 4096 00000080:00200000:3.0:1788201707.091593:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000053255732[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118887] 00000080:00200000:3.0:1788201707.091596:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002908bc01 fffffc0001029 2:0 ffff93c0c4400ee0 118887 lru 00000080:00200000:3.0:1788201707.091602:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003c55dce9 118887: 1< 2 - - > 2< 486961152 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70472 - -> 00000080:00200000:3.0:1788201707.091604:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000053255732 00000080:00200000:3.0:1788201707.091605:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.091607:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.091615:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.091626:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.091629:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.091631:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.091639:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.091642:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.091644:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.091648:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091656:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.091664:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.091666:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.091670:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.091672:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118887, 118887] 00000080:00200000:3.0:1788201707.091683:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.091686:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.091715:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.091719:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.091736:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486965248, count: 4096 00000080:00200000:3.0:1788201707.091771:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486965248, bytes: 4096 00000080:00200000:3.0:1788201707.091774:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.091778:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.091779:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486965248, 486969344) 0 0 00000080:00200000:3.0:1788201707.091782:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.091788:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118888, 118888] 00020000:00200000:3.0:1788201707.091791:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486965248 chunk: [486965248, 486969344) 486969344, 4096 00020000:00200000:3.0:1788201707.091793:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.091807:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118888, 118888] 00000008:00010000:3.0:1788201707.091813:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.091815:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.091822:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091850:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486965248 486969343) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091860:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091869:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091877:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.091885:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486965248, 486969344) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.091890:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118888 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.091894:0:17463:0:(rw26.c:898:ll_write_end()) pos 486965248, len 4096, copied 4096 00000080:00200000:3.0:1788201707.091898:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000faf33220[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118888] 00000080:00200000:3.0:1788201707.091901:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fde89798 fffffc0001029 2:0 ffff93c0c44007e0 118888 lru 00000080:00200000:3.0:1788201707.091907:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e0159a29 118888: 1< 2 - - > 2< 486965248 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70473 - -> 00000080:00200000:3.0:1788201707.091908:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000faf33220 00000080:00200000:3.0:1788201707.091909:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.091911:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.091919:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.091936:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.091938:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.091941:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.091949:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.091952:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.091954:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.091958:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.091966:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.091974:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.091977:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.091981:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.091984:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118888, 118888] 00000080:00200000:3.0:1788201707.091995:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.091998:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.092044:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.092047:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.092065:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486969344, count: 4096 00000080:00200000:3.0:1788201707.092077:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486969344, bytes: 4096 00000080:00200000:3.0:1788201707.092080:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.092083:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.092085:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486969344, 486973440) 0 0 00000080:00200000:3.0:1788201707.092087:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.092092:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118889, 118889] 00020000:00200000:3.0:1788201707.092096:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486969344 chunk: [486969344, 486973440) 486973440, 4096 00020000:00200000:3.0:1788201707.092098:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.092109:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118889, 118889] 00000008:00010000:3.0:1788201707.092114:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.092116:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.092124:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092134:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486969344 486973439) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092143:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092152:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092159:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.092166:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486969344, 486973440) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.092170:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118889 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.092175:0:17463:0:(rw26.c:898:ll_write_end()) pos 486969344, len 4096, copied 4096 00000080:00200000:3.0:1788201707.092178:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ccbe68d1[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118889] 00000080:00200000:3.0:1788201707.092181:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d2244e73 fffffc0001029 2:0 ffff93c0c44002a0 118889 lru 00000080:00200000:3.0:1788201707.092187:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000061f8eb30 118889: 1< 2 - - > 2< 486969344 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70474 - -> 00000080:00200000:3.0:1788201707.092189:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ccbe68d1 00000080:00200000:3.0:1788201707.092190:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.092192:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.092200:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.092211:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.092214:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.092216:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.092224:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.092226:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.092229:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.092233:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092241:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.092249:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.092251:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.092254:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.092257:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118889, 118889] 00000080:00200000:3.0:1788201707.092267:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.092270:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.092294:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.092297:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.092312:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486973440, count: 4096 00000080:00200000:3.0:1788201707.092321:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486973440, bytes: 4096 00000080:00200000:3.0:1788201707.092324:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.092327:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.092329:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486973440, 486977536) 0 0 00000080:00200000:3.0:1788201707.092331:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.092336:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118890, 118890] 00020000:00200000:3.0:1788201707.092339:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486973440 chunk: [486973440, 486977536) 486977536, 4096 00020000:00200000:3.0:1788201707.092342:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.092354:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118890, 118890] 00000008:00010000:3.0:1788201707.092359:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.092361:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.092369:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092377:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486973440 486977535) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092386:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092394:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092401:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.092409:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486973440, 486977536) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.092414:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118890 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.092419:0:17463:0:(rw26.c:898:ll_write_end()) pos 486973440, len 4096, copied 4096 00000080:00200000:3.0:1788201707.092423:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000044ba0fc[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118890] 00000080:00200000:3.0:1788201707.092426:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000069aea28a fffffc0001029 2:0 ffff93c0c4400e00 118890 lru 00000080:00200000:3.0:1788201707.092432:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000009526e0f 118890: 1< 2 - - > 2< 486973440 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70475 - -> 00000080:00200000:3.0:1788201707.092436:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000044ba0fc 00000080:00200000:3.0:1788201707.092437:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.092439:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.092447:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.092459:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.092462:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.092464:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.092473:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.092475:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.092477:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.092481:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092489:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.092496:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.092498:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.092502:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.092505:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118890, 118890] 00000080:00200000:3.0:1788201707.092517:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.092521:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.092550:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.092554:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.092570:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486977536, count: 4096 00000080:00200000:3.0:1788201707.092581:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486977536, bytes: 4096 00000080:00200000:3.0:1788201707.092584:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.092588:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.092589:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486977536, 486981632) 0 0 00000080:00200000:3.0:1788201707.092591:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.092596:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118891, 118891] 00020000:00200000:3.0:1788201707.092599:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486977536 chunk: [486977536, 486981632) 486981632, 4096 00020000:00200000:3.0:1788201707.092601:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.092613:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118891, 118891] 00000008:00010000:3.0:1788201707.092619:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.092621:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.092628:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092638:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486977536 486981631) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092647:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092655:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092662:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.092670:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486977536, 486981632) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.092674:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118891 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.092678:0:17463:0:(rw26.c:898:ll_write_end()) pos 486977536, len 4096, copied 4096 00000080:00200000:3.0:1788201707.092682:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000472c6cf0[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118891] 00000080:00200000:3.0:1788201707.092686:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c81d649f fffffc0001029 2:0 ffff93c0c44001c0 118891 lru 00000080:00200000:3.0:1788201707.092691:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008bbab3aa 118891: 1< 2 - - > 2< 486977536 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70476 - -> 00000080:00200000:3.0:1788201707.092693:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000472c6cf0 00000080:00200000:3.0:1788201707.092695:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.092696:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.092704:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.092715:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.092718:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.092720:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.092728:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.092731:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.092734:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.092737:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092742:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.092748:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.092749:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.092751:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.092753:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118891, 118891] 00000080:00200000:3.0:1788201707.092761:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.092763:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.092781:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.092783:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.092795:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486981632, count: 4096 00000080:00200000:3.0:1788201707.092801:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486981632, bytes: 4096 00000080:00200000:3.0:1788201707.092804:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.092807:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.092808:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486981632, 486985728) 0 0 00000080:00200000:3.0:1788201707.092811:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.092816:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118892, 118892] 00020000:00200000:3.0:1788201707.092818:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486981632 chunk: [486981632, 486985728) 486985728, 4096 00020000:00200000:3.0:1788201707.092821:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.092851:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118892, 118892] 00000008:00010000:3.0:1788201707.092857:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.092860:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.092868:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092876:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486981632 486985727) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092885:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092892:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092899:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.092907:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486981632, 486985728) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.092911:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118892 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.092917:0:17463:0:(rw26.c:898:ll_write_end()) pos 486981632, len 4096, copied 4096 00000080:00200000:3.0:1788201707.092921:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006e83b477[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118892] 00000080:00200000:3.0:1788201707.092923:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f2dbef73 fffffc0001029 2:0 ffff93c0c4400620 118892 lru 00000080:00200000:3.0:1788201707.092929:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c6cb16e4 118892: 1< 2 - - > 2< 486981632 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70477 - -> 00000080:00200000:3.0:1788201707.092931:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006e83b477 00000080:00200000:3.0:1788201707.092932:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.092934:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.092943:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.092955:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.092958:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.092960:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.092969:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.092972:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.092974:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.092978:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.092986:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.092995:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.092997:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.093001:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.093004:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118892, 118892] 00000080:00200000:3.0:1788201707.093027:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.093030:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.093060:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.093064:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.093080:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486985728, count: 4096 00000080:00200000:3.0:1788201707.093092:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486985728, bytes: 4096 00000080:00200000:3.0:1788201707.093095:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.093098:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.093100:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486985728, 486989824) 0 0 00000080:00200000:3.0:1788201707.093102:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.093107:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118893, 118893] 00020000:00200000:3.0:1788201707.093110:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486985728 chunk: [486985728, 486989824) 486989824, 4096 00020000:00200000:3.0:1788201707.093113:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.093125:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118893, 118893] 00000008:00010000:3.0:1788201707.093129:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.093131:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.093138:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093148:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486985728 486989823) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093157:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093165:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093172:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.093179:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486985728, 486989824) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.093184:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118893 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.093189:0:17463:0:(rw26.c:898:ll_write_end()) pos 486985728, len 4096, copied 4096 00000080:00200000:3.0:1788201707.093193:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ac851f30[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118893] 00000080:00200000:3.0:1788201707.093196:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000dfb07296 fffffc0001029 2:0 ffff93c0c44010e0 118893 lru 00000080:00200000:3.0:1788201707.093202:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d7e443e1 118893: 1< 2 - - > 2< 486985728 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70478 - -> 00000080:00200000:3.0:1788201707.093204:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ac851f30 00000080:00200000:3.0:1788201707.093205:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.093207:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.093215:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.093226:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.093229:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.093231:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.093239:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.093242:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.093244:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.093248:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093256:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.093264:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.093266:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.093270:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.093273:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118893, 118893] 00000080:00200000:3.0:1788201707.093284:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.093287:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.093317:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.093321:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.093339:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486989824, count: 4096 00000080:00200000:3.0:1788201707.093350:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486989824, bytes: 4096 00000080:00200000:3.0:1788201707.093353:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.093357:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.093358:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486989824, 486993920) 0 0 00000080:00200000:3.0:1788201707.093360:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.093365:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118894, 118894] 00020000:00200000:3.0:1788201707.093368:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486989824 chunk: [486989824, 486993920) 486993920, 4096 00020000:00200000:3.0:1788201707.093370:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.093382:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118894, 118894] 00000008:00010000:3.0:1788201707.093387:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.093389:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.093396:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093405:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486989824 486993919) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093414:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093421:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093428:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.093441:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486989824, 486993920) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.093446:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118894 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.093451:0:17463:0:(rw26.c:898:ll_write_end()) pos 486989824, len 4096, copied 4096 00000080:00200000:3.0:1788201707.093456:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000bd946b3f[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118894] 00000080:00200000:3.0:1788201707.093458:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a5db00de fffffc0001029 2:0 ffff93c0c4401700 118894 lru 00000080:00200000:3.0:1788201707.093464:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000806868a5 118894: 1< 2 - - > 2< 486989824 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70479 - -> 00000080:00200000:3.0:1788201707.093466:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000bd946b3f 00000080:00200000:3.0:1788201707.093467:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.093469:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.093478:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.093490:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.093493:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.093495:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.093503:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.093506:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.093509:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.093513:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093520:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.093528:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.093530:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.093535:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.093538:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118894, 118894] 00000080:00200000:3.0:1788201707.093549:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.093552:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.093581:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.093584:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.093600:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486993920, count: 4096 00000080:00200000:3.0:1788201707.093611:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486993920, bytes: 4096 00000080:00200000:3.0:1788201707.093613:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.093617:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.093618:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486993920, 486998016) 0 0 00000080:00200000:3.0:1788201707.093620:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.093625:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118895, 118895] 00020000:00200000:3.0:1788201707.093628:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486993920 chunk: [486993920, 486998016) 486998016, 4096 00020000:00200000:3.0:1788201707.093630:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.093643:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118895, 118895] 00000008:00010000:3.0:1788201707.093647:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.093650:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.093657:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093665:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486993920 486998015) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093673:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093680:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093691:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.093698:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486993920, 486998016) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.093703:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118895 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.093708:0:17463:0:(rw26.c:898:ll_write_end()) pos 486993920, len 4096, copied 4096 00000080:00200000:3.0:1788201707.093712:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000019ad3ef0[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118895] 00000080:00200000:3.0:1788201707.093714:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d4a50b60 fffffc0001029 2:0 ffff93c0c4401000 118895 lru 00000080:00200000:3.0:1788201707.093720:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000026dec36a 118895: 1< 2 - - > 2< 486993920 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70480 - -> 00000080:00200000:3.0:1788201707.093722:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000019ad3ef0 00000080:00200000:3.0:1788201707.093723:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.093725:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.093734:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.093745:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.093748:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.093750:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.093759:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.093761:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.093764:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.093767:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093774:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.093787:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.093790:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.093794:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.093797:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118895, 118895] 00000080:00200000:3.0:1788201707.093807:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.093810:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.093866:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.093870:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.093889:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 486998016, count: 4096 00000080:00200000:3.0:1788201707.093900:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 486998016, bytes: 4096 00000080:00200000:3.0:1788201707.093903:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.093907:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.093908:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [486998016, 487002112) 0 0 00000080:00200000:3.0:1788201707.093911:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.093916:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118896, 118896] 00020000:00200000:3.0:1788201707.093919:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 486998016 chunk: [486998016, 487002112) 487002112, 4096 00020000:00200000:3.0:1788201707.093921:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.093933:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118896, 118896] 00000008:00010000:3.0:1788201707.093937:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.093940:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.093947:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093963:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (486998016 487002111) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093972:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093984:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.093991:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.094000:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [486998016, 487002112) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.094005:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118896 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.094021:0:17463:0:(rw26.c:898:ll_write_end()) pos 486998016, len 4096, copied 4096 00000080:00200000:3.0:1788201707.094025:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002f4874b7[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118896] 00000080:00200000:3.0:1788201707.094028:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d59151d8 fffffc0001029 2:0 ffff93c0c4401a80 118896 lru 00000080:00200000:3.0:1788201707.094034:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007fc2e86c 118896: 1< 2 - - > 2< 486998016 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70481 - -> 00000080:00200000:3.0:1788201707.094036:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002f4874b7 00000080:00200000:3.0:1788201707.094037:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.094039:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.094048:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.094061:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.094064:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.094066:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.094075:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.094078:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.094081:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.094085:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094099:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.094107:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.094109:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.094113:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.094115:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118896, 118896] 00000080:00200000:3.0:1788201707.094126:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.094129:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.094156:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.094159:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.094176:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487002112, count: 4096 00000080:00200000:3.0:1788201707.094186:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487002112, bytes: 4096 00000080:00200000:3.0:1788201707.094189:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.094192:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.094194:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487002112, 487006208) 0 0 00000080:00200000:3.0:1788201707.094196:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.094201:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118897, 118897] 00020000:00200000:3.0:1788201707.094204:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487002112 chunk: [487002112, 487006208) 487006208, 4096 00020000:00200000:3.0:1788201707.094206:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.094217:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118897, 118897] 00000008:00010000:3.0:1788201707.094221:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.094223:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.094230:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094240:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487002112 487006207) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094255:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094263:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094270:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.094277:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487002112, 487006208) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.094281:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118897 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.094286:0:17463:0:(rw26.c:898:ll_write_end()) pos 487002112, len 4096, copied 4096 00000080:00200000:3.0:1788201707.094290:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a6e21537[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118897] 00000080:00200000:3.0:1788201707.094293:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009199cb8b fffffc0001029 2:0 ffff93c0c4401380 118897 lru 00000080:00200000:3.0:1788201707.094299:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e3459128 118897: 1< 2 - - > 2< 487002112 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70482 - -> 00000080:00200000:3.0:1788201707.094301:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a6e21537 00000080:00200000:3.0:1788201707.094302:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.094304:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.094312:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.094322:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.094325:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.094327:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.094334:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.094337:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.094340:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.094344:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094352:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.094359:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.094361:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.094365:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.094368:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118897, 118897] 00000080:00200000:3.0:1788201707.094379:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.094382:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.094410:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.094413:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.094429:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487006208, count: 4096 00000080:00200000:3.0:1788201707.094440:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487006208, bytes: 4096 00000080:00200000:3.0:1788201707.094443:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.094446:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.094447:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487006208, 487010304) 0 0 00000080:00200000:3.0:1788201707.094449:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.094454:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118898, 118898] 00020000:00200000:3.0:1788201707.094457:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487006208 chunk: [487006208, 487010304) 487010304, 4096 00020000:00200000:3.0:1788201707.094459:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.094470:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118898, 118898] 00000008:00010000:3.0:1788201707.094475:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.094477:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.094485:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094499:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487006208 487010303) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094507:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094514:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094521:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.094529:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487006208, 487010304) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.094534:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118898 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.094539:0:17463:0:(rw26.c:898:ll_write_end()) pos 487006208, len 4096, copied 4096 00000080:00200000:3.0:1788201707.094543:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008ea7fa89[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118898] 00000080:00200000:3.0:1788201707.094546:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002767fce0 fffffc0001029 2:0 ffff93c0c44018c0 118898 lru 00000080:00200000:3.0:1788201707.094551:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002846d6f2 118898: 1< 2 - - > 2< 487006208 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70483 - -> 00000080:00200000:3.0:1788201707.094554:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008ea7fa89 00000080:00200000:3.0:1788201707.094555:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.094557:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.094565:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.094576:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.094579:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.094582:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.094590:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.094593:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.094595:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.094599:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094607:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.094615:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.094617:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.094621:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.094623:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118898, 118898] 00000080:00200000:3.0:1788201707.094633:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.094636:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.094661:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.094664:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.094678:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487010304, count: 4096 00000080:00200000:3.0:1788201707.094687:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487010304, bytes: 4096 00000080:00200000:3.0:1788201707.094690:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.094693:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.094694:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487010304, 487014400) 0 0 00000080:00200000:3.0:1788201707.094697:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.094701:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118899, 118899] 00020000:00200000:3.0:1788201707.094704:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487010304 chunk: [487010304, 487014400) 487014400, 4096 00020000:00200000:3.0:1788201707.094706:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.094716:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118899, 118899] 00000008:00010000:3.0:1788201707.094721:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.094723:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.094730:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094739:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487010304 487014399) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094748:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094755:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094762:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.094769:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487010304, 487014400) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.094774:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118899 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.094779:0:17463:0:(rw26.c:898:ll_write_end()) pos 487010304, len 4096, copied 4096 00000080:00200000:3.0:1788201707.094783:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002960db75[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118899] 00000080:00200000:3.0:1788201707.094786:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cfbf8031 fffffc0001029 2:0 ffff93c0c4401ee0 118899 lru 00000080:00200000:3.0:1788201707.094792:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000342da74 118899: 1< 2 - - > 2< 487010304 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70484 - -> 00000080:00200000:3.0:1788201707.094794:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002960db75 00000080:00200000:3.0:1788201707.094795:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.094797:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.094805:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.094817:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.094820:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.094823:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.094852:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.094855:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.094858:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.094863:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.094871:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.094880:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.094882:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.094886:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.094889:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118899, 118899] 00000080:00200000:3.0:1788201707.094900:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.094904:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.094932:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.094936:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.094950:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487014400, count: 4096 00000080:00200000:3.0:1788201707.094960:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487014400, bytes: 4096 00000080:00200000:3.0:1788201707.094963:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.094967:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.094969:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487014400, 487018496) 0 0 00000080:00200000:3.0:1788201707.094970:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.094974:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118900, 118900] 00020000:00200000:3.0:1788201707.094976:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487014400 chunk: [487014400, 487018496) 487018496, 4096 00020000:00200000:3.0:1788201707.094977:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.094986:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118900, 118900] 00000008:00010000:3.0:1788201707.094990:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.094993:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.095000:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095020:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487014400 487018495) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095030:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095038:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095045:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.095053:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487014400, 487018496) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.095059:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118900 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.095064:0:17463:0:(rw26.c:898:ll_write_end()) pos 487014400, len 4096, copied 4096 00000080:00200000:3.0:1788201707.095069:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a280a82b[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118900] 00000080:00200000:3.0:1788201707.095072:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c70352f4 fffffc0001029 2:0 ffff93c0c44017e0 118900 lru 00000080:00200000:3.0:1788201707.095078:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000084208d8c 118900: 1< 2 - - > 2< 487014400 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70485 - -> 00000080:00200000:3.0:1788201707.095080:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a280a82b 00000080:00200000:3.0:1788201707.095081:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.095083:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.095092:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.095109:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.095113:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.095115:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.095124:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.095126:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.095129:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.095133:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095141:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.095150:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.095152:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.095156:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.095159:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118900, 118900] 00000080:00200000:3.0:1788201707.095170:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.095173:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.095201:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.095205:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.095222:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487018496, count: 4096 00000080:00200000:3.0:1788201707.095233:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487018496, bytes: 4096 00000080:00200000:3.0:1788201707.095236:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.095239:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.095241:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487018496, 487022592) 0 0 00000080:00200000:3.0:1788201707.095243:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.095248:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118901, 118901] 00020000:00200000:3.0:1788201707.095252:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487018496 chunk: [487018496, 487022592) 487022592, 4096 00020000:00200000:3.0:1788201707.095254:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.095267:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118901, 118901] 00000008:00010000:3.0:1788201707.095272:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.095275:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.095282:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095292:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487018496 487022591) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095300:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095307:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095314:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.095322:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487018496, 487022592) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.095327:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118901 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.095332:0:17463:0:(rw26.c:898:ll_write_end()) pos 487018496, len 4096, copied 4096 00000080:00200000:3.0:1788201707.095336:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f0260974[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118901] 00000080:00200000:3.0:1788201707.095339:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e90d517c fffffc0001029 2:0 ffff93c0c44012a0 118901 lru 00000080:00200000:3.0:1788201707.095344:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000737a49a6 118901: 1< 2 - - > 2< 487018496 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70486 - -> 00000080:00200000:3.0:1788201707.095346:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f0260974 00000080:00200000:3.0:1788201707.095347:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.095350:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.095359:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.095371:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.095373:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.095376:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.095384:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.095387:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.095389:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.095393:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095401:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.095409:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.095412:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.095416:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.095419:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118901, 118901] 00000080:00200000:3.0:1788201707.095431:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.095434:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.095464:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.095468:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.095485:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487022592, count: 4096 00000080:00200000:3.0:1788201707.095496:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487022592, bytes: 4096 00000080:00200000:3.0:1788201707.095499:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.095502:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.095504:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487022592, 487026688) 0 0 00000080:00200000:3.0:1788201707.095507:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.095512:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118902, 118902] 00020000:00200000:3.0:1788201707.095515:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487022592 chunk: [487022592, 487026688) 487026688, 4096 00020000:00200000:3.0:1788201707.095517:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.095528:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118902, 118902] 00000008:00010000:3.0:1788201707.095533:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.095535:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.095542:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095551:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487022592 487026687) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095560:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095568:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095575:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.095582:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487022592, 487026688) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.095586:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118902 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.095590:0:17463:0:(rw26.c:898:ll_write_end()) pos 487022592, len 4096, copied 4096 00000080:00200000:3.0:1788201707.095594:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fc977519[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118902] 00000080:00200000:3.0:1788201707.095596:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000433aa03e fffffc0001029 2:0 ffff93c0c13439a0 118902 lru 00000080:00200000:3.0:1788201707.095602:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c6c50756 118902: 1< 2 - - > 2< 487022592 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70487 - -> 00000080:00200000:3.0:1788201707.095606:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fc977519 00000080:00200000:3.0:1788201707.095607:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.095609:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.095616:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.095627:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.095630:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.095632:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.095640:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.095642:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.095645:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.095649:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095656:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.095663:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.095665:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.095669:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.095672:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118902, 118902] 00000080:00200000:3.0:1788201707.095684:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.095687:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.095715:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.095718:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.095735:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487026688, count: 4096 00000080:00200000:3.0:1788201707.095746:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487026688, bytes: 4096 00000080:00200000:3.0:1788201707.095749:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.095752:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.095754:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487026688, 487030784) 0 0 00000080:00200000:3.0:1788201707.095756:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.095761:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118903, 118903] 00020000:00200000:3.0:1788201707.095764:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487026688 chunk: [487026688, 487030784) 487030784, 4096 00020000:00200000:3.0:1788201707.095766:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.095778:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118903, 118903] 00000008:00010000:3.0:1788201707.095784:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.095786:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.095794:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095803:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487026688 487030783) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095812:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095820:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095827:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.095850:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487026688, 487030784) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.095856:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118903 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.095861:0:17463:0:(rw26.c:898:ll_write_end()) pos 487026688, len 4096, copied 4096 00000080:00200000:3.0:1788201707.095865:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000000406400[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118903] 00000080:00200000:3.0:1788201707.095868:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f4f61fb6 fffffc0001029 2:0 ffff93c0c1343b60 118903 lru 00000080:00200000:3.0:1788201707.095873:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e52d7124 118903: 1< 2 - - > 2< 487026688 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70488 - -> 00000080:00200000:3.0:1788201707.095875:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000000406400 00000080:00200000:3.0:1788201707.095876:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.095877:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.095886:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.095896:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.095899:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.095900:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.095909:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.095912:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.095914:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.095918:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.095925:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.095932:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.095935:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.095938:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.095941:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118903, 118903] 00000080:00200000:3.0:1788201707.095952:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.095955:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.095983:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.095986:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.096002:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487030784, count: 4096 00000080:00200000:3.0:1788201707.096024:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487030784, bytes: 4096 00000080:00200000:3.0:1788201707.096027:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.096030:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.096032:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487030784, 487034880) 0 0 00000080:00200000:3.0:1788201707.096034:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.096039:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118904, 118904] 00020000:00200000:3.0:1788201707.096042:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487030784 chunk: [487030784, 487034880) 487034880, 4096 00020000:00200000:3.0:1788201707.096044:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.096055:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118904, 118904] 00000008:00010000:3.0:1788201707.096059:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.096061:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.096068:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096078:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487030784 487034879) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096086:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096095:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096102:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.096109:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487030784, 487034880) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.096113:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118904 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.096118:0:17463:0:(rw26.c:898:ll_write_end()) pos 487030784, len 4096, copied 4096 00000080:00200000:3.0:1788201707.096122:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f0aeefed[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118904] 00000080:00200000:3.0:1788201707.096125:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000140ee99 fffffc0001029 2:0 ffff93c0c1343c40 118904 lru 00000080:00200000:3.0:1788201707.096131:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d3e20052 118904: 1< 2 - - > 2< 487030784 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70489 - -> 00000080:00200000:3.0:1788201707.096133:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f0aeefed 00000080:00200000:3.0:1788201707.096135:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.096136:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.096144:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.096155:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.096158:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.096160:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.096168:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.096170:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.096173:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.096177:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096184:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.096190:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.096192:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.096194:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.096196:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118904, 118904] 00000080:00200000:3.0:1788201707.096206:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.096210:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.096238:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.096243:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.096260:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487034880, count: 4096 00000080:00200000:3.0:1788201707.096270:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487034880, bytes: 4096 00000080:00200000:3.0:1788201707.096273:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.096276:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.096278:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487034880, 487038976) 0 0 00000080:00200000:3.0:1788201707.096280:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.096286:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118905, 118905] 00020000:00200000:3.0:1788201707.096288:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487034880 chunk: [487034880, 487038976) 487038976, 4096 00020000:00200000:3.0:1788201707.096291:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.096303:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118905, 118905] 00000008:00010000:3.0:1788201707.096309:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.096311:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.096319:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096328:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487034880 487038975) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096337:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096345:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096352:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.096359:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487034880, 487038976) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.096365:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118905 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.096370:0:17463:0:(rw26.c:898:ll_write_end()) pos 487034880, len 4096, copied 4096 00000080:00200000:3.0:1788201707.096375:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000036dec57b[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118905] 00000080:00200000:3.0:1788201707.096377:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000099672ada fffffc0001029 2:0 ffff93c0c1343540 118905 lru 00000080:00200000:3.0:1788201707.096383:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000cebd53dc 118905: 1< 2 - - > 2< 487034880 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70490 - -> 00000080:00200000:3.0:1788201707.096385:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000036dec57b 00000080:00200000:3.0:1788201707.096386:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.096389:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.096397:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.096409:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.096411:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.096413:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.096422:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.096425:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.096428:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.096432:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096440:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.096448:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.096451:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.096455:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.096457:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118905, 118905] 00000080:00200000:3.0:1788201707.096468:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.096471:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.096498:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.096501:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.096518:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487038976, count: 4096 00000080:00200000:3.0:1788201707.096529:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487038976, bytes: 4096 00000080:00200000:3.0:1788201707.096532:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.096536:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.096537:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487038976, 487043072) 0 0 00000080:00200000:3.0:1788201707.096540:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.096545:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118906, 118906] 00020000:00200000:3.0:1788201707.096548:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487038976 chunk: [487038976, 487043072) 487043072, 4096 00020000:00200000:3.0:1788201707.096550:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.096562:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118906, 118906] 00000008:00010000:3.0:1788201707.096568:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.096570:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.096577:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096587:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487038976 487043071) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096596:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096603:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096610:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.096623:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487038976, 487043072) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.096628:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118906 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.096633:0:17463:0:(rw26.c:898:ll_write_end()) pos 487038976, len 4096, copied 4096 00000080:00200000:3.0:1788201707.096637:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000048bf06f6[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118906] 00000080:00200000:3.0:1788201707.096640:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002da47a28 fffffc0001029 2:0 ffff93c0c1343460 118906 lru 00000080:00200000:3.0:1788201707.096645:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000002b8146a 118906: 1< 2 - - > 2< 487038976 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70491 - -> 00000080:00200000:3.0:1788201707.096648:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000048bf06f6 00000080:00200000:3.0:1788201707.096649:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.096651:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.096659:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.096671:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.096674:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.096676:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.096686:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.096688:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.096691:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.096695:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096704:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.096712:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.096714:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.096718:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.096721:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118906, 118906] 00000080:00200000:3.0:1788201707.096732:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.096735:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.096763:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.096767:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.096783:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487043072, count: 4096 00000080:00200000:3.0:1788201707.096793:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487043072, bytes: 4096 00000080:00200000:3.0:1788201707.096796:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.096799:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.096801:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487043072, 487047168) 0 0 00000080:00200000:3.0:1788201707.096803:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.096808:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118907, 118907] 00020000:00200000:3.0:1788201707.096811:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487043072 chunk: [487043072, 487047168) 487047168, 4096 00020000:00200000:3.0:1788201707.096813:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.096824:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118907, 118907] 00000008:00010000:3.0:1788201707.096844:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.096847:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.096855:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096864:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487043072 487047167) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096873:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096881:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096894:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.096901:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487043072, 487047168) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.096906:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118907 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.096911:0:17463:0:(rw26.c:898:ll_write_end()) pos 487043072, len 4096, copied 4096 00000080:00200000:3.0:1788201707.096914:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000df29c246[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118907] 00000080:00200000:3.0:1788201707.096917:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004cdb8a95 fffffc0001029 2:0 ffff93c0c13430e0 118907 lru 00000080:00200000:3.0:1788201707.096923:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001987411d 118907: 1< 2 - - > 2< 487043072 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70492 - -> 00000080:00200000:3.0:1788201707.096925:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000df29c246 00000080:00200000:3.0:1788201707.096926:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.096928:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.096936:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.096946:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.096949:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.096951:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.096959:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.096962:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.096964:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.096969:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.096976:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.096989:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.096991:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.096993:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.096996:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118907, 118907] 00000080:00200000:3.0:1788201707.097003:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.097005:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.097046:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.097049:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.097065:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487047168, count: 4096 00000080:00200000:3.0:1788201707.097075:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487047168, bytes: 4096 00000080:00200000:3.0:1788201707.097078:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.097082:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.097084:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487047168, 487051264) 0 0 00000080:00200000:3.0:1788201707.097086:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.097092:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118908, 118908] 00020000:00200000:3.0:1788201707.097095:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487047168 chunk: [487047168, 487051264) 487051264, 4096 00020000:00200000:3.0:1788201707.097097:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.097110:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118908, 118908] 00000008:00010000:3.0:1788201707.097116:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.097118:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.097125:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097134:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487047168 487051263) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097144:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097157:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097165:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.097172:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487047168, 487051264) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.097177:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118908 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.097181:0:17463:0:(rw26.c:898:ll_write_end()) pos 487047168, len 4096, copied 4096 00000080:00200000:3.0:1788201707.097185:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ceaaf514[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118908] 00000080:00200000:3.0:1788201707.097188:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006aecb00b fffffc0001029 2:0 ffff93c0c1343700 118908 lru 00000080:00200000:3.0:1788201707.097193:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000aef6697f 118908: 1< 2 - - > 2< 487047168 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70493 - -> 00000080:00200000:3.0:1788201707.097195:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ceaaf514 00000080:00200000:3.0:1788201707.097196:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.097198:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.097206:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.097217:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.097220:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.097222:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.097230:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.097232:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.097235:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.097239:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097253:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.097260:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.097262:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.097265:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.097268:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118908, 118908] 00000080:00200000:3.0:1788201707.097279:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.097282:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.097313:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.097317:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.097334:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487051264, count: 4096 00000080:00200000:3.0:1788201707.097344:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487051264, bytes: 4096 00000080:00200000:3.0:1788201707.097347:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.097351:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.097352:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487051264, 487055360) 0 0 00000080:00200000:3.0:1788201707.097355:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.097360:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118909, 118909] 00020000:00200000:3.0:1788201707.097363:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487051264 chunk: [487051264, 487055360) 487055360, 4096 00020000:00200000:3.0:1788201707.097365:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.097375:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118909, 118909] 00000008:00010000:3.0:1788201707.097380:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.097382:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.097389:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097398:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487051264 487055359) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097411:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097420:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097427:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.097434:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487051264, 487055360) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.097438:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118909 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.097442:0:17463:0:(rw26.c:898:ll_write_end()) pos 487051264, len 4096, copied 4096 00000080:00200000:3.0:1788201707.097446:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a12d820c[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118909] 00000080:00200000:3.0:1788201707.097449:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c087a0b7 fffffc0001029 2:0 ffff93c0c1343000 118909 lru 00000080:00200000:3.0:1788201707.097454:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f39a8810 118909: 1< 2 - - > 2< 487051264 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70494 - -> 00000080:00200000:3.0:1788201707.097456:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a12d820c 00000080:00200000:3.0:1788201707.097457:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.097459:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.097467:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.097479:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.097481:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.097483:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.097491:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.097494:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.097497:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.097501:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097508:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.097517:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.097519:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.097523:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.097526:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118909, 118909] 00000080:00200000:3.0:1788201707.097538:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.097541:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.097569:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.097572:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.097587:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487055360, count: 4096 00000080:00200000:3.0:1788201707.097597:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487055360, bytes: 4096 00000080:00200000:3.0:1788201707.097599:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.097603:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.097604:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487055360, 487059456) 0 0 00000080:00200000:3.0:1788201707.097607:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.097611:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118910, 118910] 00020000:00200000:3.0:1788201707.097614:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487055360 chunk: [487055360, 487059456) 487059456, 4096 00020000:00200000:3.0:1788201707.097616:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.097627:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118910, 118910] 00000008:00010000:3.0:1788201707.097633:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.097635:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.097644:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097657:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487055360 487059455) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097666:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097674:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097681:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.097688:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487055360, 487059456) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.097693:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118910 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.097697:0:17463:0:(rw26.c:898:ll_write_end()) pos 487055360, len 4096, copied 4096 00000080:00200000:3.0:1788201707.097701:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000c42cf51[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118910] 00000080:00200000:3.0:1788201707.097704:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000db9caa89 fffffc0001029 2:0 ffff93c0c1343a80 118910 lru 00000080:00200000:3.0:1788201707.097709:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000171d66aa 118910: 1< 2 - - > 2< 487055360 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70495 - -> 00000080:00200000:3.0:1788201707.097711:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000c42cf51 00000080:00200000:3.0:1788201707.097712:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.097714:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.097722:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.097734:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.097737:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.097739:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.097748:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.097750:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.097753:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.097757:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097765:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.097773:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.097776:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.097779:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.097782:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118910, 118910] 00000080:00200000:3.0:1788201707.097792:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.097795:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.097819:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.097822:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.097852:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487059456, count: 4096 00000080:00200000:3.0:1788201707.097863:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487059456, bytes: 4096 00000080:00200000:3.0:1788201707.097865:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.097869:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.097870:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487059456, 487063552) 0 0 00000080:00200000:3.0:1788201707.097873:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.097877:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118911, 118911] 00020000:00200000:3.0:1788201707.097880:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487059456 chunk: [487059456, 487063552) 487063552, 4096 00020000:00200000:3.0:1788201707.097883:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.097893:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118911, 118911] 00000008:00010000:3.0:1788201707.097898:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.097900:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.097907:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097916:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487059456 487063551) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097925:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097932:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097938:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.097946:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487059456, 487063552) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.097950:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118911 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.097953:0:17463:0:(rw26.c:898:ll_write_end()) pos 487059456, len 4096, copied 4096 00000080:00200000:3.0:1788201707.097956:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000034c87403[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118911] 00000080:00200000:3.0:1788201707.097958:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c3a85889 fffffc0001029 2:0 ffff93c0c1343380 118911 lru 00000080:00200000:3.0:1788201707.097961:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000d83c25a 118911: 1< 2 - - > 2< 487059456 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70496 - -> 00000080:00200000:3.0:1788201707.097963:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000034c87403 00000080:00200000:3.0:1788201707.097964:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.097965:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.097970:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.097978:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.097980:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.097982:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.097988:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.097990:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.097991:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.097994:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.097998:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.098004:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.098005:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.098020:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.098023:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118911, 118911] 00000080:00200000:3.0:1788201707.098034:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.098038:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.098066:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.098070:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.098087:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487063552, count: 4096 00000080:00200000:3.0:1788201707.098097:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487063552, bytes: 4096 00000080:00200000:3.0:1788201707.098100:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.098103:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.098105:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487063552, 487067648) 0 0 00000080:00200000:3.0:1788201707.098108:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.098112:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118912, 118912] 00020000:00200000:3.0:1788201707.098115:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487063552 chunk: [487063552, 487067648) 487067648, 4096 00020000:00200000:3.0:1788201707.098117:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.098128:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118912, 118912] 00000008:00010000:3.0:1788201707.098134:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.098136:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.098144:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098153:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487063552 487067647) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098162:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098169:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098176:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.098184:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487063552, 487067648) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.098189:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118912 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.098194:0:17463:0:(rw26.c:898:ll_write_end()) pos 487063552, len 4096, copied 4096 00000080:00200000:3.0:1788201707.098198:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005ff3d182[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118912] 00000080:00200000:3.0:1788201707.098201:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001521e4b1 fffffc0001029 2:0 ffff93c0c13438c0 118912 lru 00000080:00200000:3.0:1788201707.098207:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000171435cb 118912: 1< 2 - - > 2< 487063552 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70497 - -> 00000080:00200000:3.0:1788201707.098209:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005ff3d182 00000080:00200000:3.0:1788201707.098211:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.098212:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.098220:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.098238:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.098240:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.098242:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.098250:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.098253:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.098255:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.098259:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098268:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.098276:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.098278:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.098281:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.098284:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118912, 118912] 00000080:00200000:3.0:1788201707.098295:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.098298:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.098317:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.098321:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.098338:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487067648, count: 4096 00000080:00200000:3.0:1788201707.098349:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487067648, bytes: 4096 00000080:00200000:3.0:1788201707.098352:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.098355:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.098357:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487067648, 487071744) 0 0 00000080:00200000:3.0:1788201707.098359:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.098364:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118913, 118913] 00020000:00200000:3.0:1788201707.098367:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487067648 chunk: [487067648, 487071744) 487071744, 4096 00020000:00200000:3.0:1788201707.098369:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.098382:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118913, 118913] 00000008:00010000:3.0:1788201707.098386:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.098389:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.098396:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098405:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487067648 487071743) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098414:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098422:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098429:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.098437:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487067648, 487071744) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.098441:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118913 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.098447:0:17463:0:(rw26.c:898:ll_write_end()) pos 487067648, len 4096, copied 4096 00000080:00200000:3.0:1788201707.098451:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000068b67aeb[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118913] 00000080:00200000:3.0:1788201707.098453:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e43e1d61 fffffc0001029 2:0 ffff93c0c1343ee0 118913 lru 00000080:00200000:3.0:1788201707.098459:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ccddb143 118913: 1< 2 - - > 2< 487067648 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70498 - -> 00000080:00200000:3.0:1788201707.098461:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000068b67aeb 00000080:00200000:3.0:1788201707.098462:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.098465:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.098473:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.098486:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.098489:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.098491:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.098500:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.098503:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.098506:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.098510:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098518:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.098525:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.098528:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.098532:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.098535:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118913, 118913] 00000080:00200000:3.0:1788201707.098547:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.098550:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.098578:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.098581:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.098598:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487071744, count: 4096 00000080:00200000:3.0:1788201707.098609:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487071744, bytes: 4096 00000080:00200000:3.0:1788201707.098611:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.098615:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.098616:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487071744, 487075840) 0 0 00000080:00200000:3.0:1788201707.098619:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.098624:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118914, 118914] 00020000:00200000:3.0:1788201707.098627:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487071744 chunk: [487071744, 487075840) 487075840, 4096 00020000:00200000:3.0:1788201707.098629:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.098640:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118914, 118914] 00000008:00010000:3.0:1788201707.098645:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.098647:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.098654:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098663:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487071744 487075839) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098671:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098679:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098685:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.098693:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487071744, 487075840) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.098698:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118914 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.098703:0:17463:0:(rw26.c:898:ll_write_end()) pos 487071744, len 4096, copied 4096 00000080:00200000:3.0:1788201707.098706:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000098814641[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118914] 00000080:00200000:3.0:1788201707.098709:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c7a1c84f fffffc0001029 2:0 ffff93c0c13437e0 118914 lru 00000080:00200000:3.0:1788201707.098714:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000235490ff 118914: 1< 2 - - > 2< 487071744 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70499 - -> 00000080:00200000:3.0:1788201707.098717:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000098814641 00000080:00200000:3.0:1788201707.098718:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.098720:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.098729:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.098740:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.098743:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.098745:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.098753:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.098756:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.098759:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.098762:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098769:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.098777:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.098779:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.098783:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.098786:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118914, 118914] 00000080:00200000:3.0:1788201707.098797:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.098800:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.098844:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.098848:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.098866:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487075840, count: 4096 00000080:00200000:3.0:1788201707.098878:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487075840, bytes: 4096 00000080:00200000:3.0:1788201707.098881:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.098884:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.098886:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487075840, 487079936) 0 0 00000080:00200000:3.0:1788201707.098888:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.098894:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118915, 118915] 00020000:00200000:3.0:1788201707.098897:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487075840 chunk: [487075840, 487079936) 487079936, 4096 00020000:00200000:3.0:1788201707.098899:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.098911:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118915, 118915] 00000008:00010000:3.0:1788201707.098916:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.098918:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.098926:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098935:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487075840 487079935) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098944:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098951:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.098959:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.098967:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487075840, 487079936) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.098972:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118915 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.098978:0:17463:0:(rw26.c:898:ll_write_end()) pos 487075840, len 4096, copied 4096 00000080:00200000:3.0:1788201707.098982:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003fb41222[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118915] 00000080:00200000:3.0:1788201707.098985:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008341c216 fffffc0001029 2:0 ffff93c0c13432a0 118915 lru 00000080:00200000:3.0:1788201707.098991:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009c36bd6a 118915: 1< 2 - - > 2< 487075840 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70500 - -> 00000080:00200000:3.0:1788201707.098993:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003fb41222 00000080:00200000:3.0:1788201707.098994:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.098996:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.099005:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.099029:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.099032:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.099035:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.099044:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.099047:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.099049:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.099053:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099061:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.099069:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.099071:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.099076:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.099078:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118915, 118915] 00000080:00200000:3.0:1788201707.099088:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.099091:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.099120:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.099124:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.099142:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487079936, count: 4096 00000080:00200000:3.0:1788201707.099153:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487079936, bytes: 4096 00000080:00200000:3.0:1788201707.099156:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.099159:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.099161:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487079936, 487084032) 0 0 00000080:00200000:3.0:1788201707.099164:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.099168:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118916, 118916] 00020000:00200000:3.0:1788201707.099171:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487079936 chunk: [487079936, 487084032) 487084032, 4096 00020000:00200000:3.0:1788201707.099173:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.099184:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118916, 118916] 00000008:00010000:3.0:1788201707.099189:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.099191:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.099198:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099208:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487079936 487084031) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099217:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099224:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099231:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.099238:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487079936, 487084032) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.099243:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118916 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.099248:0:17463:0:(rw26.c:898:ll_write_end()) pos 487079936, len 4096, copied 4096 00000080:00200000:3.0:1788201707.099252:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000361abc3f[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118916] 00000080:00200000:3.0:1788201707.099255:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000003530287 fffffc0001029 2:0 ffff93c0c1343e00 118916 lru 00000080:00200000:3.0:1788201707.099261:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b2f51e90 118916: 1< 2 - - > 2< 487079936 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70501 - -> 00000080:00200000:3.0:1788201707.099263:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000361abc3f 00000080:00200000:3.0:1788201707.099264:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.099266:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.099274:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.099285:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.099287:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.099289:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.099297:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.099300:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.099302:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.099306:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099314:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.099322:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.099324:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.099327:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.099330:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118916, 118916] 00000080:00200000:3.0:1788201707.099340:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.099344:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.099370:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.099374:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.099390:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487084032, count: 4096 00000080:00200000:3.0:1788201707.099400:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487084032, bytes: 4096 00000080:00200000:3.0:1788201707.099403:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.099406:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.099408:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487084032, 487088128) 0 0 00000080:00200000:3.0:1788201707.099410:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.099415:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118917, 118917] 00020000:00200000:3.0:1788201707.099417:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487084032 chunk: [487084032, 487088128) 487088128, 4096 00020000:00200000:3.0:1788201707.099419:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.099429:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118917, 118917] 00000008:00010000:3.0:1788201707.099434:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.099436:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.099443:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099452:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487084032 487088127) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099461:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099468:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099475:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.099483:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487084032, 487088128) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.099489:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118917 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.099494:0:17463:0:(rw26.c:898:ll_write_end()) pos 487084032, len 4096, copied 4096 00000080:00200000:3.0:1788201707.099498:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000072badc7d[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118917] 00000080:00200000:3.0:1788201707.099501:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cf6dc8c6 fffffc0001029 2:0 ffff93c0c13431c0 118917 lru 00000080:00200000:3.0:1788201707.099507:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000762a1241 118917: 1< 2 - - > 2< 487084032 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70502 - -> 00000080:00200000:3.0:1788201707.099509:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000072badc7d 00000080:00200000:3.0:1788201707.099510:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.099512:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.099521:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.099534:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.099537:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.099539:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.099549:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.099551:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.099554:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.099557:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099565:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.099573:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.099576:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.099580:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.099582:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118917, 118917] 00000080:00200000:3.0:1788201707.099593:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.099596:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.099622:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.099625:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.099640:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487088128, count: 4096 00000080:00200000:3.0:1788201707.099649:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487088128, bytes: 4096 00000080:00200000:3.0:1788201707.099652:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.099655:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.099657:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487088128, 487092224) 0 0 00000080:00200000:3.0:1788201707.099659:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.099663:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118918, 118918] 00020000:00200000:3.0:1788201707.099666:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487088128 chunk: [487088128, 487092224) 487092224, 4096 00020000:00200000:3.0:1788201707.099669:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.099679:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118918, 118918] 00000008:00010000:3.0:1788201707.099684:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.099686:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.099693:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099702:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487088128 487092223) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099711:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099719:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099726:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.099738:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487088128, 487092224) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.099743:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118918 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.099747:0:17463:0:(rw26.c:898:ll_write_end()) pos 487088128, len 4096, copied 4096 00000080:00200000:3.0:1788201707.099751:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004c351b44[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118918] 00000080:00200000:3.0:1788201707.099754:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009db53f78 fffffc0001029 2:0 ffff93c0c1343620 118918 lru 00000080:00200000:3.0:1788201707.099760:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000004b43082 118918: 1< 2 - - > 2< 487088128 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70503 - -> 00000080:00200000:3.0:1788201707.099762:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004c351b44 00000080:00200000:3.0:1788201707.099763:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.099765:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.099772:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.099783:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.099785:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.099786:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.099793:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.099794:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.099796:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.099799:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099803:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.099808:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.099810:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.099813:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.099814:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118918, 118918] 00000080:00200000:3.0:1788201707.099822:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.099824:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.099861:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.099864:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.099874:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487092224, count: 4096 00000080:00200000:3.0:1788201707.099880:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487092224, bytes: 4096 00000080:00200000:3.0:1788201707.099882:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.099885:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.099886:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487092224, 487096320) 0 0 00000080:00200000:3.0:1788201707.099887:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.099890:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118919, 118919] 00020000:00200000:3.0:1788201707.099893:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487092224 chunk: [487092224, 487096320) 487096320, 4096 00020000:00200000:3.0:1788201707.099894:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.099902:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118919, 118919] 00000008:00010000:3.0:1788201707.099907:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.099910:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.099917:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099925:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487092224 487096319) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099934:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099942:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.099953:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.099958:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487092224, 487096320) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.099962:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118919 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.099966:0:17463:0:(rw26.c:898:ll_write_end()) pos 487092224, len 4096, copied 4096 00000080:00200000:3.0:1788201707.099970:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c63bb79a[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118919] 00000080:00200000:3.0:1788201707.099972:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005347f682 fffffc0001029 2:0 ffff93c0c13447e0 118919 lru 00000080:00200000:3.0:1788201707.099978:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000215951bd 118919: 1< 2 - - > 2< 487092224 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70504 - -> 00000080:00200000:3.0:1788201707.099981:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c63bb79a 00000080:00200000:3.0:1788201707.099982:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.099983:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.099992:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.100004:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.100019:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.100021:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.100031:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.100034:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.100036:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.100040:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100049:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.100063:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.100065:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.100069:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.100072:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118919, 118919] 00000080:00200000:3.0:1788201707.100084:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.100087:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.100115:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.100119:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.100136:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487096320, count: 4096 00000080:00200000:3.0:1788201707.100147:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487096320, bytes: 4096 00000080:00200000:3.0:1788201707.100150:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.100153:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.100155:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487096320, 487100416) 0 0 00000080:00200000:3.0:1788201707.100157:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.100162:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118920, 118920] 00020000:00200000:3.0:1788201707.100165:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487096320 chunk: [487096320, 487100416) 487100416, 4096 00020000:00200000:3.0:1788201707.100167:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.100179:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118920, 118920] 00000008:00010000:3.0:1788201707.100184:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.100186:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.100193:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100203:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487096320 487100415) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100212:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100225:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100232:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.100240:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487096320, 487100416) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.100245:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118920 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.100250:0:17463:0:(rw26.c:898:ll_write_end()) pos 487096320, len 4096, copied 4096 00000080:00200000:3.0:1788201707.100255:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007d5800f9[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118920] 00000080:00200000:3.0:1788201707.100258:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000dc06b873 fffffc0001029 2:0 ffff93c0c13442a0 118920 lru 00000080:00200000:3.0:1788201707.100264:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b22357ff 118920: 1< 2 - - > 2< 487096320 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70505 - -> 00000080:00200000:3.0:1788201707.100266:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007d5800f9 00000080:00200000:3.0:1788201707.100267:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.100269:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.100278:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.100291:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.100294:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.100296:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.100304:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.100307:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.100310:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.100314:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100326:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.100333:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.100336:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.100340:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.100343:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118920, 118920] 00000080:00200000:3.0:1788201707.100355:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.100358:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.100388:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.100391:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.100408:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487100416, count: 4096 00000080:00200000:3.0:1788201707.100419:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487100416, bytes: 4096 00000080:00200000:3.0:1788201707.100421:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.100425:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.100426:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487100416, 487104512) 0 0 00000080:00200000:3.0:1788201707.100429:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.100434:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118921, 118921] 00020000:00200000:3.0:1788201707.100437:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487100416 chunk: [487100416, 487104512) 487104512, 4096 00020000:00200000:3.0:1788201707.100439:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.100451:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118921, 118921] 00000008:00010000:3.0:1788201707.100457:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.100459:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.100466:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100475:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487100416 487104511) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100487:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100494:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100500:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.100508:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487100416, 487104512) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.100513:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118921 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.100518:0:17463:0:(rw26.c:898:ll_write_end()) pos 487100416, len 4096, copied 4096 00000080:00200000:3.0:1788201707.100521:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008550fc99[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118921] 00000080:00200000:3.0:1788201707.100524:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000829180e6 fffffc0001029 2:0 ffff93c0c1344e00 118921 lru 00000080:00200000:3.0:1788201707.100529:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ca94d2aa 118921: 1< 2 - - > 2< 487100416 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70506 - -> 00000080:00200000:3.0:1788201707.100531:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008550fc99 00000080:00200000:3.0:1788201707.100532:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.100533:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.100541:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.100553:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.100555:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.100558:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.100566:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.100569:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.100572:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.100576:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100583:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.100591:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.100593:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.100597:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.100600:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118921, 118921] 00000080:00200000:3.0:1788201707.100611:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.100614:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.100643:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.100646:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.100662:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487104512, count: 4096 00000080:00200000:3.0:1788201707.100673:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487104512, bytes: 4096 00000080:00200000:3.0:1788201707.100676:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.100680:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.100681:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487104512, 487108608) 0 0 00000080:00200000:3.0:1788201707.100684:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.100689:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118922, 118922] 00020000:00200000:3.0:1788201707.100692:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487104512 chunk: [487104512, 487108608) 487108608, 4096 00020000:00200000:3.0:1788201707.100694:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.100706:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118922, 118922] 00000008:00010000:3.0:1788201707.100711:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.100713:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.100720:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100734:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487104512 487108607) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100743:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100750:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100758:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.100765:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487104512, 487108608) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.100770:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118922 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.100775:0:17463:0:(rw26.c:898:ll_write_end()) pos 487104512, len 4096, copied 4096 00000080:00200000:3.0:1788201707.100779:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000006e3164e[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118922] 00000080:00200000:3.0:1788201707.100782:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cbe23203 fffffc0001029 2:0 ffff93c0c13441c0 118922 lru 00000080:00200000:3.0:1788201707.100787:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a976b4a4 118922: 1< 2 - - > 2< 487104512 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70507 - -> 00000080:00200000:3.0:1788201707.100789:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000006e3164e 00000080:00200000:3.0:1788201707.100791:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.100793:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.100801:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.100812:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.100815:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.100817:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.100827:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.100846:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.100849:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.100853:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.100860:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.100868:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.100870:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.100874:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.100878:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118922, 118922] 00000080:00200000:3.0:1788201707.100889:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.100892:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.100920:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.100924:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.100940:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487108608, count: 4096 00000080:00200000:3.0:1788201707.100951:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487108608, bytes: 4096 00000080:00200000:3.0:1788201707.100954:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.100957:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.100959:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487108608, 487112704) 0 0 00000080:00200000:3.0:1788201707.100961:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.100966:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118923, 118923] 00020000:00200000:3.0:1788201707.100969:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487108608 chunk: [487108608, 487112704) 487112704, 4096 00020000:00200000:3.0:1788201707.100971:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.100984:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118923, 118923] 00000008:00010000:3.0:1788201707.100989:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.100992:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.101000:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101020:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487108608 487112703) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101030:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101038:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101045:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.101052:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487108608, 487112704) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.101057:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118923 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.101062:0:17463:0:(rw26.c:898:ll_write_end()) pos 487108608, len 4096, copied 4096 00000080:00200000:3.0:1788201707.101066:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008427004e[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118923] 00000080:00200000:3.0:1788201707.101068:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b7b4b5fd fffffc0001029 2:0 ffff93c0c1344620 118923 lru 00000080:00200000:3.0:1788201707.101073:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000575d9e49 118923: 1< 2 - - > 2< 487108608 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70508 - -> 00000080:00200000:3.0:1788201707.101075:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008427004e 00000080:00200000:3.0:1788201707.101076:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.101078:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.101086:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.101097:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.101100:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.101102:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.101111:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.101114:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.101116:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.101120:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101128:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.101137:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.101139:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.101143:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.101147:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118923, 118923] 00000080:00200000:3.0:1788201707.101158:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.101161:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.101188:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.101192:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.101207:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487112704, count: 4096 00000080:00200000:3.0:1788201707.101217:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487112704, bytes: 4096 00000080:00200000:3.0:1788201707.101219:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.101223:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.101224:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487112704, 487116800) 0 0 00000080:00200000:3.0:1788201707.101226:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.101231:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118924, 118924] 00020000:00200000:3.0:1788201707.101234:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487112704 chunk: [487112704, 487116800) 487116800, 4096 00020000:00200000:3.0:1788201707.101236:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.101248:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118924, 118924] 00000008:00010000:3.0:1788201707.101253:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.101256:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.101264:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101273:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487112704 487116799) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101282:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101291:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101298:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.101306:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487112704, 487116800) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.101310:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118924 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.101314:0:17463:0:(rw26.c:898:ll_write_end()) pos 487112704, len 4096, copied 4096 00000080:00200000:3.0:1788201707.101318:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003613a584[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118924] 00000080:00200000:3.0:1788201707.101321:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000dd3dd7b4 fffffc0001029 2:0 ffff93c0c1344d20 118924 lru 00000080:00200000:3.0:1788201707.101326:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a13dc63d 118924: 1< 2 - - > 2< 487112704 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70509 - -> 00000080:00200000:3.0:1788201707.101328:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003613a584 00000080:00200000:3.0:1788201707.101329:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.101330:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.101338:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.101355:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.101358:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.101360:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.101369:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.101372:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.101374:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.101378:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101386:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.101393:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.101396:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.101400:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.101404:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118924, 118924] 00000080:00200000:3.0:1788201707.101415:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.101418:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.101447:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.101451:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.101467:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487116800, count: 4096 00000080:00200000:3.0:1788201707.101478:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487116800, bytes: 4096 00000080:00200000:3.0:1788201707.101481:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.101484:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.101485:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487116800, 487120896) 0 0 00000080:00200000:3.0:1788201707.101487:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.101491:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118925, 118925] 00020000:00200000:3.0:1788201707.101494:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487116800 chunk: [487116800, 487120896) 487120896, 4096 00020000:00200000:3.0:1788201707.101495:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.101507:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118925, 118925] 00000008:00010000:3.0:1788201707.101512:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.101514:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.101522:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101531:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487116800 487120895) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101539:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101547:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101553:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.101560:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487116800, 487120896) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.101565:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118925 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.101571:0:17463:0:(rw26.c:898:ll_write_end()) pos 487116800, len 4096, copied 4096 00000080:00200000:3.0:1788201707.101575:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004e28ed08[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118925] 00000080:00200000:3.0:1788201707.101578:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000995dd98c fffffc0001029 2:0 ffff93c0c13449a0 118925 lru 00000080:00200000:3.0:1788201707.101582:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003352caf0 118925: 1< 2 - - > 2< 487116800 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70510 - -> 00000080:00200000:3.0:1788201707.101584:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004e28ed08 00000080:00200000:3.0:1788201707.101585:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.101588:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.101596:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.101607:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.101610:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.101612:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.101622:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.101624:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.101627:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.101631:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101638:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.101646:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.101649:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.101653:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.101656:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118925, 118925] 00000080:00200000:3.0:1788201707.101667:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.101671:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.101700:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.101704:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.101721:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487120896, count: 4096 00000080:00200000:3.0:1788201707.101731:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487120896, bytes: 4096 00000080:00200000:3.0:1788201707.101734:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.101738:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.101739:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487120896, 487124992) 0 0 00000080:00200000:3.0:1788201707.101742:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.101747:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118926, 118926] 00020000:00200000:3.0:1788201707.101750:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487120896 chunk: [487120896, 487124992) 487124992, 4096 00020000:00200000:3.0:1788201707.101752:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.101762:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118926, 118926] 00000008:00010000:3.0:1788201707.101767:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.101769:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.101776:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101785:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487120896 487124991) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101794:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101802:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101809:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.101816:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487120896, 487124992) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.101820:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118926 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.101824:0:17463:0:(rw26.c:898:ll_write_end()) pos 487120896, len 4096, copied 4096 00000080:00200000:3.0:1788201707.101828:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003fd20a3d[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118926] 00000080:00200000:3.0:1788201707.101848:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000068cf460f fffffc0001029 2:0 ffff93c0c1344b60 118926 lru 00000080:00200000:3.0:1788201707.101854:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000b52f65c 118926: 1< 2 - - > 2< 487120896 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70511 - -> 00000080:00200000:3.0:1788201707.101857:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003fd20a3d 00000080:00200000:3.0:1788201707.101858:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.101861:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.101869:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.101881:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.101884:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.101886:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.101894:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.101897:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.101899:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.101903:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.101911:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.101919:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.101922:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.101926:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.101929:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118926, 118926] 00000080:00200000:3.0:1788201707.101940:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.101944:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.101973:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.101977:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.101992:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487124992, count: 4096 00000080:00200000:3.0:1788201707.102002:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487124992, bytes: 4096 00000080:00200000:3.0:1788201707.102005:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.102020:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.102021:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487124992, 487129088) 0 0 00000080:00200000:3.0:1788201707.102024:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.102029:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118927, 118927] 00020000:00200000:3.0:1788201707.102032:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487124992 chunk: [487124992, 487129088) 487129088, 4096 00020000:00200000:3.0:1788201707.102034:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.102046:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118927, 118927] 00000008:00010000:3.0:1788201707.102050:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.102052:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.102059:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102068:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487124992 487129087) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102077:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102084:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102089:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.102095:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487124992, 487129088) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.102100:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118927 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.102104:0:17463:0:(rw26.c:898:ll_write_end()) pos 487124992, len 4096, copied 4096 00000080:00200000:3.0:1788201707.102108:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004295e3e8[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118927] 00000080:00200000:3.0:1788201707.102112:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000b427d51 fffffc0001029 2:0 ffff93c0c1344c40 118927 lru 00000080:00200000:3.0:1788201707.102117:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000064bd8182 118927: 1< 2 - - > 2< 487124992 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70512 - -> 00000080:00200000:3.0:1788201707.102119:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004295e3e8 00000080:00200000:3.0:1788201707.102120:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.102122:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.102131:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.102144:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.102147:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.102149:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.102158:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.102161:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.102163:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.102167:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102176:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.102184:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.102186:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.102190:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.102193:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118927, 118927] 00000080:00200000:3.0:1788201707.102203:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.102207:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.102234:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.102238:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.102256:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487129088, count: 4096 00000080:00200000:3.0:1788201707.102267:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487129088, bytes: 4096 00000080:00200000:3.0:1788201707.102270:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.102274:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.102276:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487129088, 487133184) 0 0 00000080:00200000:3.0:1788201707.102278:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.102283:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118928, 118928] 00020000:00200000:3.0:1788201707.102286:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487129088 chunk: [487129088, 487133184) 487133184, 4096 00020000:00200000:3.0:1788201707.102288:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.102301:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118928, 118928] 00000008:00010000:3.0:1788201707.102307:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.102309:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.102317:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102327:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487129088 487133183) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102335:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102342:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102349:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.102357:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487129088, 487133184) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.102362:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118928 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.102368:0:17463:0:(rw26.c:898:ll_write_end()) pos 487129088, len 4096, copied 4096 00000080:00200000:3.0:1788201707.102372:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003c958e14[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118928] 00000080:00200000:3.0:1788201707.102375:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005dcdf9c4 fffffc0001029 2:0 ffff93c0c1344540 118928 lru 00000080:00200000:3.0:1788201707.102380:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a2964040 118928: 1< 2 - - > 2< 487129088 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70513 - -> 00000080:00200000:3.0:1788201707.102383:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003c958e14 00000080:00200000:3.0:1788201707.102384:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.102386:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.102394:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.102406:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.102409:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.102411:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.102420:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.102422:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.102425:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.102428:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102436:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.102444:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.102446:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.102451:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.102454:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118928, 118928] 00000080:00200000:3.0:1788201707.102465:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.102468:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.102498:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.102502:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.102518:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487133184, count: 4096 00000080:00200000:3.0:1788201707.102528:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487133184, bytes: 4096 00000080:00200000:3.0:1788201707.102531:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.102535:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.102536:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487133184, 487137280) 0 0 00000080:00200000:3.0:1788201707.102539:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.102544:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118929, 118929] 00020000:00200000:3.0:1788201707.102546:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487133184 chunk: [487133184, 487137280) 487137280, 4096 00020000:00200000:3.0:1788201707.102549:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.102559:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118929, 118929] 00000008:00010000:3.0:1788201707.102564:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.102566:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.102573:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102582:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487133184 487137279) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102590:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102597:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102605:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.102613:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487133184, 487137280) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.102618:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118929 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.102623:0:17463:0:(rw26.c:898:ll_write_end()) pos 487133184, len 4096, copied 4096 00000080:00200000:3.0:1788201707.102627:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d0040df3[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118929] 00000080:00200000:3.0:1788201707.102629:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a798794a fffffc0001029 2:0 ffff93c0c1344460 118929 lru 00000080:00200000:3.0:1788201707.102636:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001f8c1a9a 118929: 1< 2 - - > 2< 487133184 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70514 - -> 00000080:00200000:3.0:1788201707.102638:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d0040df3 00000080:00200000:3.0:1788201707.102639:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.102641:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.102649:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.102660:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.102662:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.102664:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.102673:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.102675:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.102677:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.102682:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102689:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.102697:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.102699:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.102702:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.102705:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118929, 118929] 00000080:00200000:3.0:1788201707.102715:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.102718:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.102744:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.102747:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.102764:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487137280, count: 4096 00000080:00200000:3.0:1788201707.102775:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487137280, bytes: 4096 00000080:00200000:3.0:1788201707.102778:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.102781:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.102783:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487137280, 487141376) 0 0 00000080:00200000:3.0:1788201707.102785:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.102790:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118930, 118930] 00020000:00200000:3.0:1788201707.102793:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487137280 chunk: [487137280, 487141376) 487141376, 4096 00020000:00200000:3.0:1788201707.102795:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.102805:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118930, 118930] 00000008:00010000:3.0:1788201707.102810:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.102812:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.102819:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102828:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487137280 487141375) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102854:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102862:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102869:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.102881:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487137280, 487141376) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.102887:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118930 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.102892:0:17463:0:(rw26.c:898:ll_write_end()) pos 487137280, len 4096, copied 4096 00000080:00200000:3.0:1788201707.102896:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001b6a28e8[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118930] 00000080:00200000:3.0:1788201707.102899:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bea78d5e fffffc0001029 2:0 ffff93c0c13440e0 118930 lru 00000080:00200000:3.0:1788201707.102905:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000020aac117 118930: 1< 2 - - > 2< 487137280 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70515 - -> 00000080:00200000:3.0:1788201707.102907:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001b6a28e8 00000080:00200000:3.0:1788201707.102908:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.102910:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.102918:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.102932:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.102934:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.102937:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.102945:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.102948:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.102950:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.102955:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.102962:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.102970:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.102973:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.102977:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.102979:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118930, 118930] 00000080:00200000:3.0:1788201707.102990:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.102994:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.103035:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.103039:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.103055:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487141376, count: 4096 00000080:00200000:3.0:1788201707.103064:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487141376, bytes: 4096 00000080:00200000:3.0:1788201707.103067:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.103071:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.103072:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487141376, 487145472) 0 0 00000080:00200000:3.0:1788201707.103075:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.103079:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118931, 118931] 00020000:00200000:3.0:1788201707.103083:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487141376 chunk: [487141376, 487145472) 487145472, 4096 00020000:00200000:3.0:1788201707.103085:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.103095:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118931, 118931] 00000008:00010000:3.0:1788201707.103100:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.103102:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.103109:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103118:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487141376 487145471) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103126:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103134:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103147:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.103155:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487141376, 487145472) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.103160:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118931 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.103166:0:17463:0:(rw26.c:898:ll_write_end()) pos 487141376, len 4096, copied 4096 00000080:00200000:3.0:1788201707.103170:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008d79bcf4[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118931] 00000080:00200000:3.0:1788201707.103173:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000033cc417b fffffc0001029 2:0 ffff93c0c1344700 118931 lru 00000080:00200000:3.0:1788201707.103178:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000070a56781 118931: 1< 2 - - > 2< 487141376 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70516 - -> 00000080:00200000:3.0:1788201707.103180:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008d79bcf4 00000080:00200000:3.0:1788201707.103181:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.103183:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.103192:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.103204:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.103206:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.103209:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.103217:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.103219:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.103222:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.103226:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103235:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.103248:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.103251:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.103254:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.103257:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118931, 118931] 00000080:00200000:3.0:1788201707.103267:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.103270:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.103296:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.103300:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.103316:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487145472, count: 4096 00000080:00200000:3.0:1788201707.103327:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487145472, bytes: 4096 00000080:00200000:3.0:1788201707.103330:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.103334:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.103335:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487145472, 487149568) 0 0 00000080:00200000:3.0:1788201707.103338:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.103343:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118932, 118932] 00020000:00200000:3.0:1788201707.103345:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487145472 chunk: [487145472, 487149568) 487149568, 4096 00020000:00200000:3.0:1788201707.103348:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.103360:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118932, 118932] 00000008:00010000:3.0:1788201707.103366:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.103368:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.103376:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103385:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487145472 487149567) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103393:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103406:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103412:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.103418:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487145472, 487149568) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.103422:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118932 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.103426:0:17463:0:(rw26.c:898:ll_write_end()) pos 487145472, len 4096, copied 4096 00000080:00200000:3.0:1788201707.103431:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004f359021[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118932] 00000080:00200000:3.0:1788201707.103433:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000200400e fffffc0001029 2:0 ffff93c0c1344000 118932 lru 00000080:00200000:3.0:1788201707.103439:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e0f2f7a4 118932: 1< 2 - - > 2< 487145472 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70517 - -> 00000080:00200000:3.0:1788201707.103441:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004f359021 00000080:00200000:3.0:1788201707.103442:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.103444:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.103452:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.103464:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.103467:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.103469:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.103480:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.103483:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.103485:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.103490:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103502:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.103509:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.103511:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.103515:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.103517:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118932, 118932] 00000080:00200000:3.0:1788201707.103529:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.103532:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.103560:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.103564:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.103580:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487149568, count: 4096 00000080:00200000:3.0:1788201707.103591:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487149568, bytes: 4096 00000080:00200000:3.0:1788201707.103594:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.103597:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.103598:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487149568, 487153664) 0 0 00000080:00200000:3.0:1788201707.103601:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.103606:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118933, 118933] 00020000:00200000:3.0:1788201707.103609:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487149568 chunk: [487149568, 487153664) 487153664, 4096 00020000:00200000:3.0:1788201707.103611:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.103623:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118933, 118933] 00000008:00010000:3.0:1788201707.103629:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.103631:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.103638:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103648:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487149568 487153663) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103661:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103668:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103674:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.103681:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487149568, 487153664) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.103686:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118933 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.103691:0:17463:0:(rw26.c:898:ll_write_end()) pos 487149568, len 4096, copied 4096 00000080:00200000:3.0:1788201707.103695:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000069c6eaa[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118933] 00000080:00200000:3.0:1788201707.103698:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b89fc0d9 fffffc0001029 2:0 ffff93c0c1344a80 118933 lru 00000080:00200000:3.0:1788201707.103703:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000287b0933 118933: 1< 2 - - > 2< 487149568 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70518 - -> 00000080:00200000:3.0:1788201707.103705:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000069c6eaa 00000080:00200000:3.0:1788201707.103707:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.103708:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.103717:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.103729:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.103732:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.103734:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.103743:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.103746:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.103748:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.103752:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103760:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.103769:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.103772:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.103776:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.103779:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118933, 118933] 00000080:00200000:3.0:1788201707.103790:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.103793:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.103823:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.103827:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.103861:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487153664, count: 4096 00000080:00200000:3.0:1788201707.103873:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487153664, bytes: 4096 00000080:00200000:3.0:1788201707.103876:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.103880:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.103881:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487153664, 487157760) 0 0 00000080:00200000:3.0:1788201707.103883:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.103889:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118934, 118934] 00020000:00200000:3.0:1788201707.103892:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487153664 chunk: [487153664, 487157760) 487157760, 4096 00020000:00200000:3.0:1788201707.103894:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.103906:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118934, 118934] 00000008:00010000:3.0:1788201707.103911:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.103913:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.103920:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103934:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487153664 487157759) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103943:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103950:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.103956:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.103962:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487153664, 487157760) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.103965:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118934 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.103969:0:17463:0:(rw26.c:898:ll_write_end()) pos 487153664, len 4096, copied 4096 00000080:00200000:3.0:1788201707.103972:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ae78ccf5[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118934] 00000080:00200000:3.0:1788201707.103975:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000be6b3d4f fffffc0001029 2:0 ffff93c0c1344380 118934 lru 00000080:00200000:3.0:1788201707.103981:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000084646514 118934: 1< 2 - - > 2< 487153664 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70519 - -> 00000080:00200000:3.0:1788201707.103983:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ae78ccf5 00000080:00200000:3.0:1788201707.103984:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.103986:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.103994:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.104018:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.104021:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.104024:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.104035:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.104037:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.104040:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.104044:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104052:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.104060:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.104063:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.104067:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.104070:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118934, 118934] 00000080:00200000:3.0:1788201707.104081:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.104084:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.104112:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.104116:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.104132:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487157760, count: 4096 00000080:00200000:3.0:1788201707.104143:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487157760, bytes: 4096 00000080:00200000:3.0:1788201707.104146:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.104150:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.104151:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487157760, 487161856) 0 0 00000080:00200000:3.0:1788201707.104153:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.104159:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118935, 118935] 00020000:00200000:3.0:1788201707.104162:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487157760 chunk: [487157760, 487161856) 487161856, 4096 00020000:00200000:3.0:1788201707.104164:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.104175:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118935, 118935] 00000008:00010000:3.0:1788201707.104180:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.104182:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.104190:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104199:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487157760 487161855) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104208:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104216:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104223:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.104231:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487157760, 487161856) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.104236:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118935 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.104241:0:17463:0:(rw26.c:898:ll_write_end()) pos 487157760, len 4096, copied 4096 00000080:00200000:3.0:1788201707.104245:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000de281a21[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118935] 00000080:00200000:3.0:1788201707.104247:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005d609c4a fffffc0001029 2:0 ffff93c0c13448c0 118935 lru 00000080:00200000:3.0:1788201707.104252:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009e791993 118935: 1< 2 - - > 2< 487157760 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70520 - -> 00000080:00200000:3.0:1788201707.104255:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000de281a21 00000080:00200000:3.0:1788201707.104256:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.104258:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.104266:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.104279:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.104282:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.104284:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.104293:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.104295:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.104298:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.104302:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104311:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.104319:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.104321:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.104324:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.104327:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118935, 118935] 00000080:00200000:3.0:1788201707.104338:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.104341:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.104369:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.104373:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.104389:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487161856, count: 4096 00000080:00200000:3.0:1788201707.104400:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487161856, bytes: 4096 00000080:00200000:3.0:1788201707.104403:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.104406:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.104408:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487161856, 487165952) 0 0 00000080:00200000:3.0:1788201707.104410:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.104415:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118936, 118936] 00020000:00200000:3.0:1788201707.104418:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487161856 chunk: [487161856, 487165952) 487165952, 4096 00020000:00200000:3.0:1788201707.104420:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.104431:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118936, 118936] 00000008:00010000:3.0:1788201707.104437:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.104440:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.104447:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104456:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487161856 487165951) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104464:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104471:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104479:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.104486:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487161856, 487165952) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.104490:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118936 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.104495:0:17463:0:(rw26.c:898:ll_write_end()) pos 487161856, len 4096, copied 4096 00000080:00200000:3.0:1788201707.104499:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d6ce016e[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118936] 00000080:00200000:3.0:1788201707.104502:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008afe4560 fffffc0001029 2:0 ffff93c0c1344ee0 118936 lru 00000080:00200000:3.0:1788201707.104507:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000515f7f1f 118936: 1< 2 - - > 2< 487161856 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70521 - -> 00000080:00200000:3.0:1788201707.104509:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d6ce016e 00000080:00200000:3.0:1788201707.104511:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.104512:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.104520:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.104536:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.104539:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.104542:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.104551:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.104554:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.104557:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.104561:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104569:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.104578:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.104580:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.104584:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.104587:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118936, 118936] 00000080:00200000:3.0:1788201707.104597:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.104600:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.104626:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.104629:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.104644:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487165952, count: 4096 00000080:00200000:3.0:1788201707.104653:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487165952, bytes: 4096 00000080:00200000:3.0:1788201707.104655:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.104659:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.104660:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487165952, 487170048) 0 0 00000080:00200000:3.0:1788201707.104662:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.104667:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118937, 118937] 00020000:00200000:3.0:1788201707.104670:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487165952 chunk: [487165952, 487170048) 487170048, 4096 00020000:00200000:3.0:1788201707.104672:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.104682:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118937, 118937] 00000008:00010000:3.0:1788201707.104686:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.104689:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.104695:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104704:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487165952 487170047) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104713:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104722:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104730:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.104737:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487165952, 487170048) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.104741:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118937 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.104745:0:17463:0:(rw26.c:898:ll_write_end()) pos 487165952, len 4096, copied 4096 00000080:00200000:3.0:1788201707.104749:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a02048a9[3 0000000075f4bf38 2 1 000000005a169ee3 0 0 118937] 00000080:00200000:3.0:1788201707.104751:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000014b5fec2 fffffc0001029 2:0 ffff93c0c13457e0 118937 lru 00000080:00200000:3.0:1788201707.104756:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000276584c4 118937: 1< 2 - - > 2< 487165952 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70522 - -> 00000080:00200000:3.0:1788201707.104758:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a02048a9 00000080:00200000:3.0:1788201707.104759:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.104761:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.104769:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.104781:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.104784:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.104786:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.104795:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.104798:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.104800:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.104804:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.104811:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.104819:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3f38 (ffff93c1d864f2d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.104821:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f2d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.104825:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.104852:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118937, 118937] 00000080:00200000:3.0:1788201707.104865:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.104868:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.104942:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.104947:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.104968:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487170048, count: 4096 00000080:00200000:2.0:1788201707.104983:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487170048, bytes: 4096 00000080:00200000:2.0:1788201707.104985:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.104989:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.104991:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487170048, 487174144) 0 0 00000080:00200000:2.0:1788201707.104994:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.105000:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118938, 118938] 00020000:00200000:2.0:1788201707.105004:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487170048 chunk: [487170048, 487174144) 487174144, 4096 00020000:00200000:2.0:1788201707.105018:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.105038:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118938, 118938] 00000008:00010000:2.0:1788201707.105046:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.105049:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.105057:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105069:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487170048 487174143) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105076:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105081:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105086:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.105091:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487170048, 487174144) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.105095:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118938 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.105100:0:17463:0:(rw26.c:898:ll_write_end()) pos 487170048, len 4096, copied 4096 00000080:00200000:2.0:1788201707.105103:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002ee0636a[3 0000000075f4bf38 2 1 00000000cfda8c55 0 0 118938] 00000080:00200000:2.0:1788201707.105104:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e44d6f5f fffffc0001029 2:0 ffff93c0c13452a0 118938 lru 00000080:00200000:2.0:1788201707.105108:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007c755bf8 118938: 1< 2 - - > 2< 487170048 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70523 - -> 00000080:00200000:2.0:1788201707.105110:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002ee0636a 00000080:00200000:2.0:1788201707.105110:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.105112:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.105119:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.105133:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.105135:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.105136:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.105146:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.105148:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.105150:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.105152:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105158:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.105163:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.105164:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.105168:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.105171:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118938, 118938] 00000080:00200000:2.0:1788201707.105182:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.105184:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.105219:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.105222:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.105234:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487174144, count: 4096 00000080:00200000:2.0:1788201707.105242:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487174144, bytes: 4096 00000080:00200000:2.0:1788201707.105246:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.105249:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.105251:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487174144, 487178240) 0 0 00000080:00200000:2.0:1788201707.105253:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.105258:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118939, 118939] 00020000:00200000:2.0:1788201707.105262:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487174144 chunk: [487174144, 487178240) 487178240, 4096 00020000:00200000:2.0:1788201707.105264:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.105277:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118939, 118939] 00000008:00010000:2.0:1788201707.105283:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.105286:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.105293:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105299:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487174144 487178239) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105305:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105310:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105314:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.105319:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487174144, 487178240) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.105323:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118939 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.105328:0:17463:0:(rw26.c:898:ll_write_end()) pos 487174144, len 4096, copied 4096 00000080:00200000:2.0:1788201707.105333:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000034d36189[3 0000000075f4bf38 2 1 00000000cfda8c55 0 0 118939] 00000080:00200000:2.0:1788201707.105335:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000069cf7ed3 fffffc0001029 2:0 ffff93c0c1345e00 118939 lru 00000080:00200000:2.0:1788201707.105341:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e41cb053 118939: 1< 2 - - > 2< 487174144 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70524 - -> 00000080:00200000:2.0:1788201707.105342:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000034d36189 00000080:00200000:2.0:1788201707.105343:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.105346:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.105355:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.105369:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.105371:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.105373:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.105381:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.105383:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.105385:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.105387:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105393:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.105398:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.105399:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.105402:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.105404:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118939, 118939] 00000080:00200000:2.0:1788201707.105411:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.105413:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.105432:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.105435:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.105445:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487178240, count: 4096 00000080:00200000:2.0:1788201707.105451:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487178240, bytes: 4096 00000080:00200000:2.0:1788201707.105453:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.105456:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.105457:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487178240, 487182336) 0 0 00000080:00200000:2.0:1788201707.105459:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.105464:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118940, 118940] 00020000:00200000:2.0:1788201707.105467:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487178240 chunk: [487178240, 487182336) 487182336, 4096 00020000:00200000:2.0:1788201707.105469:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.105483:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118940, 118940] 00000008:00010000:2.0:1788201707.105488:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.105498:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.105507:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105517:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487178240 487182335) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105525:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105533:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105539:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.105547:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487178240, 487182336) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.105553:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118940 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.105559:0:17463:0:(rw26.c:898:ll_write_end()) pos 487178240, len 4096, copied 4096 00000080:00200000:2.0:1788201707.105563:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000389163e7[3 0000000075f4bf38 2 1 00000000cfda8c55 0 0 118940] 00000080:00200000:2.0:1788201707.105566:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007837088c fffffc0001029 2:0 ffff93c0c13451c0 118940 lru 00000080:00200000:2.0:1788201707.105571:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d8d6af65 118940: 1< 2 - - > 2< 487178240 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70525 - -> 00000080:00200000:2.0:1788201707.105574:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000389163e7 00000080:00200000:2.0:1788201707.105575:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.105577:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.105585:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.105597:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.105600:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.105602:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.105611:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.105614:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.105616:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.105620:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105626:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.105634:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.105636:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.105641:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.105644:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118940, 118940] 00000080:00200000:2.0:1788201707.105655:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.105658:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.105683:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.105686:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.105699:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487182336, count: 4096 00000080:00200000:2.0:1788201707.105710:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487182336, bytes: 4096 00000080:00200000:2.0:1788201707.105714:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.105718:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.105719:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487182336, 487186432) 0 0 00000080:00200000:2.0:1788201707.105721:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.105727:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118941, 118941] 00020000:00200000:2.0:1788201707.105730:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487182336 chunk: [487182336, 487186432) 487186432, 4096 00020000:00200000:2.0:1788201707.105732:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.105745:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118941, 118941] 00000008:00010000:2.0:1788201707.105751:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.105753:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.105761:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105769:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487182336 487186431) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105775:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105783:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105790:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.105798:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487182336, 487186432) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.105803:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118941 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.105808:0:17463:0:(rw26.c:898:ll_write_end()) pos 487182336, len 4096, copied 4096 00000080:00200000:2.0:1788201707.105813:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000271a269a[3 0000000075f4bf38 2 1 00000000cfda8c55 0 0 118941] 00000080:00200000:2.0:1788201707.105816:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d28c3bcf fffffc0001029 2:0 ffff93c0c1345620 118941 lru 00000080:00200000:2.0:1788201707.105821:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000614f520f 118941: 1< 2 - - > 2< 487182336 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70526 - -> 00000080:00200000:2.0:1788201707.105823:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000271a269a 00000080:00200000:2.0:1788201707.105824:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.105826:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.105854:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.105867:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.105870:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.105872:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.105882:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.105885:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.105887:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.105891:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.105899:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.105908:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.105910:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.105915:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.105918:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118941, 118941] 00000080:00200000:2.0:1788201707.105926:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.105928:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.105949:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.105951:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.105962:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487186432, count: 4096 00000080:00200000:2.0:1788201707.105968:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487186432, bytes: 4096 00000080:00200000:2.0:1788201707.105970:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.105973:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.105974:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487186432, 487190528) 0 0 00000080:00200000:2.0:1788201707.105975:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.105979:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118942, 118942] 00020000:00200000:2.0:1788201707.105981:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487186432 chunk: [487186432, 487190528) 487190528, 4096 00020000:00200000:2.0:1788201707.105982:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.105991:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118942, 118942] 00000008:00010000:2.0:1788201707.105996:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.105999:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.106035:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106046:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487186432 487190527) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106054:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106059:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106064:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.106069:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487186432, 487190528) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.106073:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118942 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.106077:0:17463:0:(rw26.c:898:ll_write_end()) pos 487186432, len 4096, copied 4096 00000080:00200000:2.0:1788201707.106080:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000fa217f19[3 0000000075f4bf38 2 1 00000000cfda8c55 0 0 118942] 00000080:00200000:2.0:1788201707.106082:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001b8822b4 fffffc0001029 2:0 ffff93c0c1345d20 118942 lru 00000080:00200000:2.0:1788201707.106086:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007f25897d 118942: 1< 2 - - > 2< 487186432 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70527 - -> 00000080:00200000:2.0:1788201707.106087:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000fa217f19 00000080:00200000:2.0:1788201707.106088:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.106089:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.106096:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.106104:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.106106:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.106107:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.106114:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.106116:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.106118:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.106120:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106125:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.106130:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.106132:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.106134:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.106136:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118942, 118942] 00000080:00200000:2.0:1788201707.106143:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.106145:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.106164:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.106166:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.106177:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487190528, count: 4096 00000080:00200000:2.0:1788201707.106183:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487190528, bytes: 4096 00000080:00200000:2.0:1788201707.106185:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.106187:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.106188:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487190528, 487194624) 0 0 00000080:00200000:2.0:1788201707.106189:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.106193:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118943, 118943] 00020000:00200000:2.0:1788201707.106195:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487190528 chunk: [487190528, 487194624) 487194624, 4096 00020000:00200000:2.0:1788201707.106196:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.106203:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118943, 118943] 00000008:00010000:2.0:1788201707.106207:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.106208:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.106213:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106219:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487190528 487194623) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106225:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106229:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106233:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.106239:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487190528, 487194624) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.106244:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118943 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.106249:0:17463:0:(rw26.c:898:ll_write_end()) pos 487190528, len 4096, copied 4096 00000080:00200000:2.0:1788201707.106253:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009f12c76d[3 0000000075f4bf38 2 1 00000000cfda8c55 0 0 118943] 00000080:00200000:2.0:1788201707.106256:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000451b04d1 fffffc0001029 2:0 ffff93c0c13459a0 118943 lru 00000080:00200000:2.0:1788201707.106262:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000045bc81fc 118943: 1< 2 - - > 2< 487190528 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70528 - -> 00000080:00200000:2.0:1788201707.106264:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009f12c76d 00000080:00200000:2.0:1788201707.106265:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.106267:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.106275:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.106287:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.106291:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.106292:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.106299:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.106301:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.106303:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.106306:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106311:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.106316:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.106317:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.106320:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.106323:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118943, 118943] 00000080:00200000:2.0:1788201707.106333:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.106337:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.106369:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.106373:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.106385:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487194624, count: 4096 00000080:00200000:2.0:1788201707.106392:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487194624, bytes: 4096 00000080:00200000:2.0:1788201707.106394:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.106396:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.106397:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487194624, 487198720) 0 0 00000080:00200000:2.0:1788201707.106399:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.106403:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118944, 118944] 00020000:00200000:2.0:1788201707.106405:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487194624 chunk: [487194624, 487198720) 487198720, 4096 00020000:00200000:2.0:1788201707.106407:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.106414:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118944, 118944] 00000008:00010000:2.0:1788201707.106417:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.106419:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.106425:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106431:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487194624 487198719) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106438:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106443:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106447:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.106455:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487194624, 487198720) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.106458:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118944 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.106462:0:17463:0:(rw26.c:898:ll_write_end()) pos 487194624, len 4096, copied 4096 00000080:00200000:2.0:1788201707.106464:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000055611413[3 0000000075f4bf38 2 1 00000000cfda8c55 0 0 118944] 00000080:00200000:2.0:1788201707.106466:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007e4875d3 fffffc0001029 2:0 ffff93c0c1345b60 118944 lru 00000080:00200000:2.0:1788201707.106470:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f614bb55 118944: 1< 2 - - > 2< 487194624 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70529 - -> 00000080:00200000:2.0:1788201707.106472:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000055611413 00000080:00200000:2.0:1788201707.106472:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.106474:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.106479:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.106487:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.106489:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.106491:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.106497:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.106499:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.106500:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.106503:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106507:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.106513:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.106515:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.106520:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.106523:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118944, 118944] 00000080:00200000:2.0:1788201707.106532:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.106535:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.106564:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.106568:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.106581:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487198720, count: 4096 00000080:00200000:2.0:1788201707.106588:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487198720, bytes: 4096 00000080:00200000:2.0:1788201707.106590:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.106593:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.106594:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487198720, 487202816) 0 0 00000080:00200000:2.0:1788201707.106595:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.106599:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118945, 118945] 00020000:00200000:2.0:1788201707.106601:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487198720 chunk: [487198720, 487202816) 487202816, 4096 00020000:00200000:2.0:1788201707.106602:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.106609:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118945, 118945] 00000008:00010000:2.0:1788201707.106612:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.106614:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.106619:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106625:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487198720 487202815) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106631:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106635:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106644:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.106648:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487198720, 487202816) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.106651:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118945 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.106654:0:17463:0:(rw26.c:898:ll_write_end()) pos 487198720, len 4096, copied 4096 00000080:00200000:2.0:1788201707.106657:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006e01ee71[3 0000000075f4bf38 2 1 00000000cfda8c55 0 0 118945] 00000080:00200000:2.0:1788201707.106659:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000914c6102 fffffc0001029 2:0 ffff93c0c1345c40 118945 lru 00000080:00200000:2.0:1788201707.106662:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000f7c1f07 118945: 1< 2 - - > 2< 487198720 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70530 - -> 00000080:00200000:2.0:1788201707.106664:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006e01ee71 00000080:00200000:2.0:1788201707.106664:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.106666:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.106671:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.106679:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.106681:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.106682:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.106688:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.106690:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.106691:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.106694:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106699:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.106708:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.106709:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.106712:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.106714:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118945, 118945] 00000080:00200000:2.0:1788201707.106721:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.106723:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.106740:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.106743:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.106752:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487202816, count: 4096 00000080:00200000:2.0:1788201707.106758:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487202816, bytes: 4096 00000080:00200000:2.0:1788201707.106760:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.106762:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.106763:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487202816, 487206912) 0 0 00000080:00200000:2.0:1788201707.106765:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.106768:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118946, 118946] 00020000:00200000:2.0:1788201707.106770:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487202816 chunk: [487202816, 487206912) 487206912, 4096 00020000:00200000:2.0:1788201707.106771:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.106778:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118946, 118946] 00000008:00010000:2.0:1788201707.106781:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.106782:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.106787:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106793:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487202816 487206911) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106798:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106807:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106811:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.106815:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487202816, 487206912) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.106817:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118946 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.106821:0:17463:0:(rw26.c:898:ll_write_end()) pos 487202816, len 4096, copied 4096 00000080:00200000:2.0:1788201707.106823:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000076008f70[3 0000000075f4bf38 2 1 00000000cfda8c55 0 0 118946] 00000080:00200000:2.0:1788201707.106825:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006836ea1b fffffc0001029 2:0 ffff93c0c1345540 118946 lru 00000080:00200000:2.0:1788201707.106841:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008ba320ef 118946: 1< 2 - - > 2< 487202816 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70531 - -> 00000080:00200000:2.0:1788201707.106843:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000076008f70 00000080:00200000:2.0:1788201707.106844:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.106845:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.106851:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.106859:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.106861:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.106862:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.106868:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.106870:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.106871:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.106874:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106882:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.106887:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.106888:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.106890:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.106892:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118946, 118946] 00000080:00200000:2.0:1788201707.106899:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.106901:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.106918:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.106920:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.106930:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487206912, count: 4096 00000080:00200000:2.0:1788201707.106936:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487206912, bytes: 4096 00000080:00200000:2.0:1788201707.106938:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.106940:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.106941:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487206912, 487211008) 0 0 00000080:00200000:2.0:1788201707.106943:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.106946:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118947, 118947] 00020000:00200000:2.0:1788201707.106948:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487206912 chunk: [487206912, 487211008) 487211008, 4096 00020000:00200000:2.0:1788201707.106949:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.106955:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118947, 118947] 00000008:00010000:2.0:1788201707.106958:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.106960:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.106965:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106970:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487206912 487211007) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106979:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106984:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.106989:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.106996:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487206912, 487211008) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.107001:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118947 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.107005:0:17463:0:(rw26.c:898:ll_write_end()) pos 487206912, len 4096, copied 4096 00000080:00200000:2.0:1788201707.107021:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000858c53d5[3 0000000075f4bf38 2 1 00000000cfda8c55 0 0 118947] 00000080:00200000:2.0:1788201707.107024:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000092a5e258 fffffc0001029 2:0 ffff93c0c1345460 118947 lru 00000080:00200000:2.0:1788201707.107030:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a8151799 118947: 1< 2 - - > 2< 487206912 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70532 - -> 00000080:00200000:2.0:1788201707.107032:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000858c53d5 00000080:00200000:2.0:1788201707.107033:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.107036:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.107045:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.107056:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.107058:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.107059:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.107066:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.107068:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.107070:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.107073:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107078:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.107083:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.107084:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.107087:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.107088:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118947, 118947] 00000080:00200000:2.0:1788201707.107095:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.107097:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.107116:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.107118:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.107128:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487211008, count: 4096 00000080:00200000:2.0:1788201707.107134:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487211008, bytes: 4096 00000080:00200000:2.0:1788201707.107136:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.107138:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.107139:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487211008, 487215104) 0 0 00000080:00200000:2.0:1788201707.107141:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.107144:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118948, 118948] 00020000:00200000:2.0:1788201707.107146:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487211008 chunk: [487211008, 487215104) 487215104, 4096 00020000:00200000:2.0:1788201707.107148:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.107154:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118948, 118948] 00000008:00010000:2.0:1788201707.107158:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.107159:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.107164:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107173:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487211008 487215103) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107179:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107183:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107187:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.107191:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487211008, 487215104) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.107195:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118948 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.107198:0:17463:0:(rw26.c:898:ll_write_end()) pos 487211008, len 4096, copied 4096 00000080:00200000:2.0:1788201707.107201:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d298f133[3 0000000075f4bf38 2 1 00000000cfda8c55 0 0 118948] 00000080:00200000:2.0:1788201707.107203:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000275779d0 fffffc0001029 2:0 ffff93c0c13450e0 118948 lru 00000080:00200000:2.0:1788201707.107207:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d49b4a2b 118948: 1< 2 - - > 2< 487211008 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70533 - -> 00000080:00200000:2.0:1788201707.107208:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d298f133 00000080:00200000:2.0:1788201707.107208:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.107209:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.107215:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.107222:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.107223:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.107225:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.107231:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.107233:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.107234:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.107237:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107245:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.107253:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.107255:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.107260:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.107263:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118948, 118948] 00000080:00200000:2.0:1788201707.107273:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.107276:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.107300:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.107303:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.107313:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487215104, count: 4096 00000080:00200000:2.0:1788201707.107319:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487215104, bytes: 4096 00000080:00200000:2.0:1788201707.107322:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.107325:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.107327:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487215104, 487219200) 0 0 00000080:00200000:2.0:1788201707.107329:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.107334:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118949, 118949] 00020000:00200000:2.0:1788201707.107337:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487215104 chunk: [487215104, 487219200) 487219200, 4096 00020000:00200000:2.0:1788201707.107339:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.107351:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118949, 118949] 00000008:00010000:2.0:1788201707.107356:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.107359:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.107367:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107375:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487215104 487219199) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107381:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107386:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107390:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.107394:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487215104, 487219200) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.107398:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118949 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.107402:0:17463:0:(rw26.c:898:ll_write_end()) pos 487215104, len 4096, copied 4096 00000080:00200000:2.0:1788201707.107404:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000024fd84f6[3 0000000075f4bf38 2 1 00000000cfda8c55 0 0 118949] 00000080:00200000:2.0:1788201707.107406:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fb440d17 fffffc0001029 2:0 ffff93c0c1345700 118949 lru 00000080:00200000:2.0:1788201707.107409:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000019c5bbae 118949: 1< 2 - - > 2< 487215104 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70534 - -> 00000080:00200000:2.0:1788201707.107410:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000024fd84f6 00000080:00200000:2.0:1788201707.107411:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.107412:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.107418:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.107426:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.107428:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.107430:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.107436:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.107438:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.107439:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.107441:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107446:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.107452:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.107453:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.107456:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.107458:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118949, 118949] 00000080:00200000:2.0:1788201707.107465:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.107467:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.107484:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.107487:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.107497:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487219200, count: 4096 00000080:00200000:2.0:1788201707.107503:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487219200, bytes: 4096 00000080:00200000:2.0:1788201707.107505:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.107507:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.107508:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487219200, 487223296) 0 0 00000080:00200000:2.0:1788201707.107510:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.107513:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118950, 118950] 00020000:00200000:2.0:1788201707.107516:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487219200 chunk: [487219200, 487223296) 487223296, 4096 00020000:00200000:2.0:1788201707.107518:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.107529:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118950, 118950] 00000008:00010000:2.0:1788201707.107535:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.107537:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.107544:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107553:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487219200 487223295) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107562:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107569:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107576:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.107584:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487219200, 487223296) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.107589:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118950 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.107594:0:17463:0:(rw26.c:898:ll_write_end()) pos 487219200, len 4096, copied 4096 00000080:00200000:2.0:1788201707.107597:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b53921bc[3 0000000075f4bf38 2 1 00000000cfda8c55 0 0 118950] 00000080:00200000:2.0:1788201707.107599:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000250a9d7c fffffc0001029 2:0 ffff93c0c1345000 118950 lru 00000080:00200000:2.0:1788201707.107602:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000be583bc4 118950: 1< 2 - - > 2< 487219200 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70535 - -> 00000080:00200000:2.0:1788201707.107604:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b53921bc 00000080:00200000:2.0:1788201707.107605:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.107606:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.107612:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.107623:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.107625:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.107626:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.107633:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.107634:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.107636:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.107639:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107644:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.107649:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.107650:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.107652:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.107654:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118950, 118950] 00000080:00200000:2.0:1788201707.107662:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.107663:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.107681:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.107684:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.107694:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487223296, count: 4096 00000080:00200000:2.0:1788201707.107700:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487223296, bytes: 4096 00000080:00200000:2.0:1788201707.107702:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.107704:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.107705:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487223296, 487227392) 0 0 00000080:00200000:2.0:1788201707.107707:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.107710:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118951, 118951] 00020000:00200000:2.0:1788201707.107713:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487223296 chunk: [487223296, 487227392) 487227392, 4096 00020000:00200000:2.0:1788201707.107714:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.107721:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118951, 118951] 00000008:00010000:2.0:1788201707.107724:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.107726:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.107733:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107741:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487223296 487227391) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107750:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107758:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107765:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.107771:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487223296, 487227392) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.107775:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118951 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.107779:0:17463:0:(rw26.c:898:ll_write_end()) pos 487223296, len 4096, copied 4096 00000080:00200000:2.0:1788201707.107781:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f175c928[3 0000000075f4bf38 2 1 00000000cfda8c55 0 0 118951] 00000080:00200000:2.0:1788201707.107783:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008720a060 fffffc0001029 2:0 ffff93c0c1345a80 118951 lru 00000080:00200000:2.0:1788201707.107787:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000af78d76d 118951: 1< 2 - - > 2< 487223296 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70536 - -> 00000080:00200000:2.0:1788201707.107788:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f175c928 00000080:00200000:2.0:1788201707.107789:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.107791:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.107797:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.107804:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.107806:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.107808:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.107814:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.107816:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.107817:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.107819:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.107824:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.107849:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c57024e8 (ffff93c1d300a128/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.107850:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a128/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.107853:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.107855:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118951, 118951] 00000080:00200000:2.0:1788201707.107863:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.107865:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.107919:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.107925:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.107949:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487227392, count: 4096 00000080:00200000:3.0:1788201707.107966:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487227392, bytes: 4096 00000080:00200000:3.0:1788201707.107969:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.107974:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.107975:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487227392, 487231488) 0 0 00000080:00200000:3.0:1788201707.107979:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.107985:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118952, 118952] 00020000:00200000:3.0:1788201707.107988:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487227392 chunk: [487227392, 487231488) 487231488, 4096 00020000:00200000:3.0:1788201707.107991:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.108006:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118952, 118952] 00000008:00010000:3.0:1788201707.108023:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.108026:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.108035:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108046:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487227392 487231487) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108056:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108064:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108072:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.108079:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487227392, 487231488) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.108084:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118952 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.108089:0:17463:0:(rw26.c:898:ll_write_end()) pos 487227392, len 4096, copied 4096 00000080:00200000:3.0:1788201707.108093:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000aae7518c[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118952] 00000080:00200000:3.0:1788201707.108096:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bec3bdf6 fffffc0001029 2:0 ffff93c0c1345380 118952 lru 00000080:00200000:3.0:1788201707.108103:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000be49fed1 118952: 1< 2 - - > 2< 487227392 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70537 - -> 00000080:00200000:3.0:1788201707.108106:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000aae7518c 00000080:00200000:3.0:1788201707.108108:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.108109:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.108118:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.108130:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.108133:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.108135:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.108145:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.108148:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.108151:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.108155:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108162:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.108170:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.108173:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.108177:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.108180:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118952, 118952] 00000080:00200000:3.0:1788201707.108197:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.108200:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.108234:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.108238:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.108254:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487231488, count: 4096 00000080:00200000:3.0:1788201707.108264:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487231488, bytes: 4096 00000080:00200000:3.0:1788201707.108267:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.108271:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.108273:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487231488, 487235584) 0 0 00000080:00200000:3.0:1788201707.108275:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.108280:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118953, 118953] 00020000:00200000:3.0:1788201707.108283:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487231488 chunk: [487231488, 487235584) 487235584, 4096 00020000:00200000:3.0:1788201707.108285:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.108296:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118953, 118953] 00000008:00010000:3.0:1788201707.108301:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.108304:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.108311:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108321:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487231488 487235583) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108331:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108339:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108346:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.108353:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487231488, 487235584) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.108357:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118953 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.108363:0:17463:0:(rw26.c:898:ll_write_end()) pos 487231488, len 4096, copied 4096 00000080:00200000:3.0:1788201707.108367:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008e05ec4e[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118953] 00000080:00200000:3.0:1788201707.108370:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001393694f fffffc0001029 2:0 ffff93c0c13458c0 118953 lru 00000080:00200000:3.0:1788201707.108376:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000048d39d62 118953: 1< 2 - - > 2< 487231488 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70538 - -> 00000080:00200000:3.0:1788201707.108378:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008e05ec4e 00000080:00200000:3.0:1788201707.108379:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.108381:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.108390:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.108401:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.108403:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.108406:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.108416:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.108418:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.108420:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.108424:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108431:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.108440:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.108442:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.108446:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.108449:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118953, 118953] 00000080:00200000:3.0:1788201707.108460:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.108463:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.108492:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.108496:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.108513:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487235584, count: 4096 00000080:00200000:3.0:1788201707.108524:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487235584, bytes: 4096 00000080:00200000:3.0:1788201707.108527:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.108531:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.108532:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487235584, 487239680) 0 0 00000080:00200000:3.0:1788201707.108535:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.108540:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118954, 118954] 00020000:00200000:3.0:1788201707.108543:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487235584 chunk: [487235584, 487239680) 487239680, 4096 00020000:00200000:3.0:1788201707.108545:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.108557:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118954, 118954] 00000008:00010000:3.0:1788201707.108562:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.108564:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.108572:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108581:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487235584 487239679) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108591:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108611:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108619:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.108627:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487235584, 487239680) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.108632:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118954 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.108637:0:17463:0:(rw26.c:898:ll_write_end()) pos 487235584, len 4096, copied 4096 00000080:00200000:3.0:1788201707.108641:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000acdc3363[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118954] 00000080:00200000:3.0:1788201707.108644:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000dcf05a40 fffffc0001029 2:0 ffff93c0c1345ee0 118954 lru 00000080:00200000:3.0:1788201707.108651:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000532de516 118954: 1< 2 - - > 2< 487235584 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70539 - -> 00000080:00200000:3.0:1788201707.108653:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000acdc3363 00000080:00200000:3.0:1788201707.108654:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.108656:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.108664:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.108676:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.108679:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.108681:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.108690:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.108693:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.108696:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.108700:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108709:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.108717:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.108719:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.108723:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.108725:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118954, 118954] 00000080:00200000:3.0:1788201707.108737:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.108739:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.108765:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.108769:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.108783:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487239680, count: 4096 00000080:00200000:3.0:1788201707.108792:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487239680, bytes: 4096 00000080:00200000:3.0:1788201707.108795:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.108799:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.108800:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487239680, 487243776) 0 0 00000080:00200000:3.0:1788201707.108803:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.108807:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118955, 118955] 00020000:00200000:3.0:1788201707.108810:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487239680 chunk: [487239680, 487243776) 487243776, 4096 00020000:00200000:3.0:1788201707.108813:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.108823:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118955, 118955] 00000008:00010000:3.0:1788201707.108828:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.108847:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.108854:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108864:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487239680 487243775) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108874:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108881:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108887:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.108895:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487239680, 487243776) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.108900:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118955 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.108904:0:17463:0:(rw26.c:898:ll_write_end()) pos 487239680, len 4096, copied 4096 00000080:00200000:3.0:1788201707.108908:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f0f3d6d6[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118955] 00000080:00200000:3.0:1788201707.108911:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b244c823 fffffc0001029 2:0 ffff93c0c13469a0 118955 lru 00000080:00200000:3.0:1788201707.108916:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001e2a9801 118955: 1< 2 - - > 2< 487239680 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70540 - -> 00000080:00200000:3.0:1788201707.108919:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f0f3d6d6 00000080:00200000:3.0:1788201707.108920:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.108922:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.108930:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.108942:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.108945:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.108947:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.108957:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.108960:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.108962:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.108966:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.108974:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.108982:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.108984:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.108988:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.108991:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118955, 118955] 00000080:00200000:3.0:1788201707.109001:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.109005:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.109045:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.109049:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.109066:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487243776, count: 4096 00000080:00200000:3.0:1788201707.109077:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487243776, bytes: 4096 00000080:00200000:3.0:1788201707.109080:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.109084:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.109085:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487243776, 487247872) 0 0 00000080:00200000:3.0:1788201707.109088:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.109093:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118956, 118956] 00020000:00200000:3.0:1788201707.109096:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487243776 chunk: [487243776, 487247872) 487247872, 4096 00020000:00200000:3.0:1788201707.109098:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.109109:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118956, 118956] 00000008:00010000:3.0:1788201707.109114:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.109116:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.109123:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109133:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487243776 487247871) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109142:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109150:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109157:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.109170:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487243776, 487247872) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.109174:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118956 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.109180:0:17463:0:(rw26.c:898:ll_write_end()) pos 487243776, len 4096, copied 4096 00000080:00200000:3.0:1788201707.109183:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002d5acb5c[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118956] 00000080:00200000:3.0:1788201707.109186:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000422b4bf2 fffffc0001029 2:0 ffff93c0c1346b60 118956 lru 00000080:00200000:3.0:1788201707.109192:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006b74e745 118956: 1< 2 - - > 2< 487243776 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70541 - -> 00000080:00200000:3.0:1788201707.109194:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002d5acb5c 00000080:00200000:3.0:1788201707.109195:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.109197:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.109206:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.109217:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.109220:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.109222:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.109232:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.109235:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.109237:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.109242:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109249:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.109257:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.109260:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.109265:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.109268:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118956, 118956] 00000080:00200000:3.0:1788201707.109279:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.109282:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.109311:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.109314:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.109331:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487247872, count: 4096 00000080:00200000:3.0:1788201707.109342:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487247872, bytes: 4096 00000080:00200000:3.0:1788201707.109345:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.109349:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.109350:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487247872, 487251968) 0 0 00000080:00200000:3.0:1788201707.109353:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.109358:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118957, 118957] 00020000:00200000:3.0:1788201707.109361:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487247872 chunk: [487247872, 487251968) 487251968, 4096 00020000:00200000:3.0:1788201707.109363:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.109374:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118957, 118957] 00000008:00010000:3.0:1788201707.109379:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.109382:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.109389:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109397:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487247872 487251967) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109406:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109413:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109425:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.109433:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487247872, 487251968) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.109438:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118957 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.109443:0:17463:0:(rw26.c:898:ll_write_end()) pos 487247872, len 4096, copied 4096 00000080:00200000:3.0:1788201707.109447:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e2e387f0[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118957] 00000080:00200000:3.0:1788201707.109450:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000af95b067 fffffc0001029 2:0 ffff93c0c1346c40 118957 lru 00000080:00200000:3.0:1788201707.109456:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000019879b33 118957: 1< 2 - - > 2< 487247872 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70542 - -> 00000080:00200000:3.0:1788201707.109458:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e2e387f0 00000080:00200000:3.0:1788201707.109459:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.109461:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.109470:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.109482:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.109485:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.109488:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.109498:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.109500:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.109503:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.109507:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109515:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.109529:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.109532:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.109536:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.109539:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118957, 118957] 00000080:00200000:3.0:1788201707.109550:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.109553:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.109581:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.109585:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.109601:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487251968, count: 4096 00000080:00200000:3.0:1788201707.109613:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487251968, bytes: 4096 00000080:00200000:3.0:1788201707.109615:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.109619:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.109620:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487251968, 487256064) 0 0 00000080:00200000:3.0:1788201707.109622:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.109628:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118958, 118958] 00020000:00200000:3.0:1788201707.109630:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487251968 chunk: [487251968, 487256064) 487256064, 4096 00020000:00200000:3.0:1788201707.109633:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.109644:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118958, 118958] 00000008:00010000:3.0:1788201707.109649:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.109652:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.109659:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109668:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487251968 487256063) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109678:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109692:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109699:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.109706:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487251968, 487256064) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.109710:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118958 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.109715:0:17463:0:(rw26.c:898:ll_write_end()) pos 487251968, len 4096, copied 4096 00000080:00200000:3.0:1788201707.109718:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f0ade398[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118958] 00000080:00200000:3.0:1788201707.109721:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000aa56aef5 fffffc0001029 2:0 ffff93c0c4401e00 118958 lru 00000080:00200000:3.0:1788201707.109727:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000855fca87 118958: 1< 2 - - > 2< 487251968 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70543 - -> 00000080:00200000:3.0:1788201707.109729:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f0ade398 00000080:00200000:3.0:1788201707.109730:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.109732:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.109740:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.109751:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.109753:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.109755:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.109763:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.109766:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.109768:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.109772:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109786:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.109794:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.109796:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.109799:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.109801:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118958, 118958] 00000080:00200000:3.0:1788201707.109812:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.109815:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.109864:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.109868:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.109880:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487256064, count: 4096 00000080:00200000:3.0:1788201707.109890:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487256064, bytes: 4096 00000080:00200000:3.0:1788201707.109893:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.109896:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.109898:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487256064, 487260160) 0 0 00000080:00200000:3.0:1788201707.109901:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.109906:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118959, 118959] 00020000:00200000:3.0:1788201707.109908:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487256064 chunk: [487256064, 487260160) 487260160, 4096 00020000:00200000:3.0:1788201707.109910:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.109923:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118959, 118959] 00000008:00010000:3.0:1788201707.109929:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.109931:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.109938:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109948:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487256064 487260159) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109962:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109970:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.109977:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.109984:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487256064, 487260160) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.109988:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118959 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.109993:0:17463:0:(rw26.c:898:ll_write_end()) pos 487256064, len 4096, copied 4096 00000080:00200000:3.0:1788201707.109997:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a2e06974[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118959] 00000080:00200000:3.0:1788201707.110000:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f38022c0 fffffc0001029 2:0 ffff93c0c44011c0 118959 lru 00000080:00200000:3.0:1788201707.110019:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000beb10f7 118959: 1< 2 - - > 2< 487256064 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70544 - -> 00000080:00200000:3.0:1788201707.110021:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a2e06974 00000080:00200000:3.0:1788201707.110023:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.110024:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.110033:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.110045:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.110047:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.110049:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.110059:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.110062:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.110065:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.110069:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110078:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.110086:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.110089:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.110093:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.110095:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118959, 118959] 00000080:00200000:3.0:1788201707.110107:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.110110:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.110135:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.110138:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.110153:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487260160, count: 4096 00000080:00200000:3.0:1788201707.110162:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487260160, bytes: 4096 00000080:00200000:3.0:1788201707.110165:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.110168:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.110170:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487260160, 487264256) 0 0 00000080:00200000:3.0:1788201707.110172:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.110176:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118960, 118960] 00020000:00200000:3.0:1788201707.110179:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487260160 chunk: [487260160, 487264256) 487264256, 4096 00020000:00200000:3.0:1788201707.110180:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.110192:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118960, 118960] 00000008:00010000:3.0:1788201707.110196:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.110199:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.110207:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110221:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487260160 487264255) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110230:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110238:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110245:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.110253:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487260160, 487264256) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.110258:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118960 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.110263:0:17463:0:(rw26.c:898:ll_write_end()) pos 487260160, len 4096, copied 4096 00000080:00200000:3.0:1788201707.110267:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007262f4fa[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118960] 00000080:00200000:3.0:1788201707.110270:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000058f3b6ed fffffc0001029 2:0 ffff93c0c4401620 118960 lru 00000080:00200000:3.0:1788201707.110275:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a84ce248 118960: 1< 2 - - > 2< 487260160 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70545 - -> 00000080:00200000:3.0:1788201707.110278:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007262f4fa 00000080:00200000:3.0:1788201707.110279:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.110281:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.110289:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.110301:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.110304:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.110307:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.110316:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.110319:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.110322:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.110326:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110334:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.110342:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.110345:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.110349:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.110353:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118960, 118960] 00000080:00200000:3.0:1788201707.110364:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.110367:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.110398:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.110402:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.110418:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487264256, count: 4096 00000080:00200000:3.0:1788201707.110429:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487264256, bytes: 4096 00000080:00200000:3.0:1788201707.110432:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.110435:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.110437:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487264256, 487268352) 0 0 00000080:00200000:3.0:1788201707.110440:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.110445:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118961, 118961] 00020000:00200000:3.0:1788201707.110448:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487264256 chunk: [487264256, 487268352) 487268352, 4096 00020000:00200000:3.0:1788201707.110450:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.110463:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118961, 118961] 00000008:00010000:3.0:1788201707.110468:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.110471:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.110479:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110487:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487264256 487268351) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110494:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110501:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110508:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.110515:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487264256, 487268352) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.110520:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118961 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.110526:0:17463:0:(rw26.c:898:ll_write_end()) pos 487264256, len 4096, copied 4096 00000080:00200000:3.0:1788201707.110530:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f26a25c6[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118961] 00000080:00200000:3.0:1788201707.110533:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ce5fc6a4 fffffc0001029 2:0 ffff93c0c4401d20 118961 lru 00000080:00200000:3.0:1788201707.110539:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000618b8860 118961: 1< 2 - - > 2< 487264256 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70546 - -> 00000080:00200000:3.0:1788201707.110541:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f26a25c6 00000080:00200000:3.0:1788201707.110542:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.110544:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.110553:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.110566:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.110569:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.110571:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.110581:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.110583:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.110586:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.110590:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110597:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.110605:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.110607:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.110611:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.110614:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118961, 118961] 00000080:00200000:3.0:1788201707.110625:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.110629:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.110656:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.110659:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.110675:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487268352, count: 4096 00000080:00200000:3.0:1788201707.110686:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487268352, bytes: 4096 00000080:00200000:3.0:1788201707.110688:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.110691:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.110693:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487268352, 487272448) 0 0 00000080:00200000:3.0:1788201707.110695:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.110700:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118962, 118962] 00020000:00200000:3.0:1788201707.110703:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487268352 chunk: [487268352, 487272448) 487272448, 4096 00020000:00200000:3.0:1788201707.110705:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.110716:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118962, 118962] 00000008:00010000:3.0:1788201707.110721:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.110724:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.110731:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110739:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487268352 487272447) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110747:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110754:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110760:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.110767:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487268352, 487272448) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.110772:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118962 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.110777:0:17463:0:(rw26.c:898:ll_write_end()) pos 487268352, len 4096, copied 4096 00000080:00200000:3.0:1788201707.110781:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f8b02132[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118962] 00000080:00200000:3.0:1788201707.110784:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000653f9634 fffffc0001029 2:0 ffff93c0c44019a0 118962 lru 00000080:00200000:3.0:1788201707.110789:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e643c3ff 118962: 1< 2 - - > 2< 487268352 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70547 - -> 00000080:00200000:3.0:1788201707.110791:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f8b02132 00000080:00200000:3.0:1788201707.110792:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.110794:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.110803:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.110819:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.110822:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.110824:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.110851:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.110854:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.110856:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.110860:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.110867:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.110875:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.110877:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.110881:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.110884:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118962, 118962] 00000080:00200000:3.0:1788201707.110895:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.110898:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.110927:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.110932:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.110948:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487272448, count: 4096 00000080:00200000:3.0:1788201707.110959:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487272448, bytes: 4096 00000080:00200000:3.0:1788201707.110962:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.110966:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.110968:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487272448, 487276544) 0 0 00000080:00200000:3.0:1788201707.110970:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.110976:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118963, 118963] 00020000:00200000:3.0:1788201707.110980:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487272448 chunk: [487272448, 487276544) 487276544, 4096 00020000:00200000:3.0:1788201707.110982:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.110995:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118963, 118963] 00000008:00010000:3.0:1788201707.111000:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.111003:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.111023:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111032:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487272448 487276543) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111041:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111049:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111056:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.111064:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487272448, 487276544) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.111069:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118963 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.111074:0:17463:0:(rw26.c:898:ll_write_end()) pos 487272448, len 4096, copied 4096 00000080:00200000:3.0:1788201707.111078:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b9cd0a61[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118963] 00000080:00200000:3.0:1788201707.111081:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007dd1f8ab fffffc0001029 2:0 ffff93c0c4401b60 118963 lru 00000080:00200000:3.0:1788201707.111086:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000edf32649 118963: 1< 2 - - > 2< 487272448 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70548 - -> 00000080:00200000:3.0:1788201707.111088:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b9cd0a61 00000080:00200000:3.0:1788201707.111089:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.111092:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.111100:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.111111:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.111114:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.111117:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.111127:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.111130:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.111132:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.111136:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111145:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.111154:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.111156:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.111160:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.111164:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118963, 118963] 00000080:00200000:3.0:1788201707.111175:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.111179:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.111208:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.111211:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.111226:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487276544, count: 4096 00000080:00200000:3.0:1788201707.111236:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487276544, bytes: 4096 00000080:00200000:3.0:1788201707.111239:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.111242:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.111243:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487276544, 487280640) 0 0 00000080:00200000:3.0:1788201707.111246:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.111251:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118964, 118964] 00020000:00200000:3.0:1788201707.111254:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487276544 chunk: [487276544, 487280640) 487280640, 4096 00020000:00200000:3.0:1788201707.111256:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.111268:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118964, 118964] 00000008:00010000:3.0:1788201707.111273:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.111276:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.111283:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111292:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487276544 487280639) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111301:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111308:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111315:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.111323:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487276544, 487280640) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.111328:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118964 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.111333:0:17463:0:(rw26.c:898:ll_write_end()) pos 487276544, len 4096, copied 4096 00000080:00200000:3.0:1788201707.111337:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008e0cbb4b[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118964] 00000080:00200000:3.0:1788201707.111340:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000027d4184e fffffc0001029 2:0 ffff93c0c4401c40 118964 lru 00000080:00200000:3.0:1788201707.111346:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000017d1cefa 118964: 1< 2 - - > 2< 487276544 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70549 - -> 00000080:00200000:3.0:1788201707.111350:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008e0cbb4b 00000080:00200000:3.0:1788201707.111351:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.111352:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.111361:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.111372:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.111375:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.111377:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.111386:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.111387:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.111389:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.111392:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111398:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.111406:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.111408:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.111412:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.111416:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118964, 118964] 00000080:00200000:3.0:1788201707.111428:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.111431:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.111461:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.111464:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.111482:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487280640, count: 4096 00000080:00200000:3.0:1788201707.111493:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487280640, bytes: 4096 00000080:00200000:3.0:1788201707.111496:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.111499:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.111501:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487280640, 487284736) 0 0 00000080:00200000:3.0:1788201707.111503:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.111509:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118965, 118965] 00020000:00200000:3.0:1788201707.111512:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487280640 chunk: [487280640, 487284736) 487284736, 4096 00020000:00200000:3.0:1788201707.111514:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.111525:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118965, 118965] 00000008:00010000:3.0:1788201707.111531:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.111533:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.111540:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111550:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487280640 487284735) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111558:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111566:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111573:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.111580:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487280640, 487284736) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.111585:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118965 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.111590:0:17463:0:(rw26.c:898:ll_write_end()) pos 487280640, len 4096, copied 4096 00000080:00200000:3.0:1788201707.111594:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000851ef510[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118965] 00000080:00200000:3.0:1788201707.111597:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d641fff6 fffffc0001029 2:0 ffff93c0c4401540 118965 lru 00000080:00200000:3.0:1788201707.111603:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000aaee5bb3 118965: 1< 2 - - > 2< 487280640 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70550 - -> 00000080:00200000:3.0:1788201707.111605:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000851ef510 00000080:00200000:3.0:1788201707.111607:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.111608:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.111616:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.111627:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.111630:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.111632:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.111640:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.111643:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.111645:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.111649:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111657:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.111665:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.111667:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.111671:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.111674:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118965, 118965] 00000080:00200000:3.0:1788201707.111686:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.111689:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.111719:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.111722:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.111738:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487284736, count: 4096 00000080:00200000:3.0:1788201707.111748:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487284736, bytes: 4096 00000080:00200000:3.0:1788201707.111751:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.111755:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.111756:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487284736, 487288832) 0 0 00000080:00200000:3.0:1788201707.111759:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.111763:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118966, 118966] 00020000:00200000:3.0:1788201707.111766:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487284736 chunk: [487284736, 487288832) 487288832, 4096 00020000:00200000:3.0:1788201707.111769:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.111780:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118966, 118966] 00000008:00010000:3.0:1788201707.111785:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.111787:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.111794:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111801:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487284736 487288831) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111809:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111817:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111824:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.111850:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487284736, 487288832) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.111857:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118966 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.111862:0:17463:0:(rw26.c:898:ll_write_end()) pos 487284736, len 4096, copied 4096 00000080:00200000:3.0:1788201707.111867:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c13ae26f[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118966] 00000080:00200000:3.0:1788201707.111870:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b0d8eb5a fffffc0001029 2:0 ffff93c0c4401460 118966 lru 00000080:00200000:3.0:1788201707.111876:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b18b7a80 118966: 1< 2 - - > 2< 487284736 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70551 - -> 00000080:00200000:3.0:1788201707.111878:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c13ae26f 00000080:00200000:3.0:1788201707.111879:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.111881:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.111889:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.111901:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.111903:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.111905:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.111914:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.111916:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.111919:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.111923:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.111931:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.111939:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.111941:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.111944:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.111947:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118966, 118966] 00000080:00200000:3.0:1788201707.111958:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.111961:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.111991:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.111995:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.112022:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487288832, count: 4096 00000080:00200000:3.0:1788201707.112034:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487288832, bytes: 4096 00000080:00200000:3.0:1788201707.112037:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.112041:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.112042:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487288832, 487292928) 0 0 00000080:00200000:3.0:1788201707.112045:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.112050:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118967, 118967] 00020000:00200000:3.0:1788201707.112053:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487288832 chunk: [487288832, 487292928) 487292928, 4096 00020000:00200000:3.0:1788201707.112055:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.112066:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118967, 118967] 00000008:00010000:3.0:1788201707.112070:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.112073:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.112080:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112089:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487288832 487292927) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112098:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112106:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112113:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.112120:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487288832, 487292928) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.112125:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118967 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.112129:0:17463:0:(rw26.c:898:ll_write_end()) pos 487288832, len 4096, copied 4096 00000080:00200000:3.0:1788201707.112133:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002a70ad86[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118967] 00000080:00200000:3.0:1788201707.112136:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ad7a337a fffffc0001029 2:0 ffff93c0c4402c40 118967 lru 00000080:00200000:3.0:1788201707.112141:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000052ad95a 118967: 1< 2 - - > 2< 487288832 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70552 - -> 00000080:00200000:3.0:1788201707.112142:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002a70ad86 00000080:00200000:3.0:1788201707.112143:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.112144:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.112151:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.112163:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.112166:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.112168:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.112178:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.112180:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.112183:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.112187:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112195:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.112203:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.112205:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.112210:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.112213:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118967, 118967] 00000080:00200000:3.0:1788201707.112224:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.112227:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.112257:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.112261:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.112278:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487292928, count: 4096 00000080:00200000:3.0:1788201707.112289:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487292928, bytes: 4096 00000080:00200000:3.0:1788201707.112292:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.112296:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.112297:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487292928, 487297024) 0 0 00000080:00200000:3.0:1788201707.112300:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.112304:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118968, 118968] 00020000:00200000:3.0:1788201707.112307:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487292928 chunk: [487292928, 487297024) 487297024, 4096 00020000:00200000:3.0:1788201707.112310:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.112320:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118968, 118968] 00000008:00010000:3.0:1788201707.112326:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.112328:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.112336:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112344:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487292928 487297023) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112352:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112360:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112367:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.112380:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487292928, 487297024) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.112385:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118968 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.112391:0:17463:0:(rw26.c:898:ll_write_end()) pos 487292928, len 4096, copied 4096 00000080:00200000:3.0:1788201707.112395:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000089471863[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118968] 00000080:00200000:3.0:1788201707.112397:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b867a04e fffffc0001029 2:0 ffff93c0c4402540 118968 lru 00000080:00200000:3.0:1788201707.112403:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007191f4d9 118968: 1< 2 - - > 2< 487292928 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70553 - -> 00000080:00200000:3.0:1788201707.112405:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000089471863 00000080:00200000:3.0:1788201707.112406:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.112408:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.112416:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.112428:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.112431:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.112433:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.112443:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.112446:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.112448:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.112452:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112460:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.112467:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.112469:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.112474:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.112477:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118968, 118968] 00000080:00200000:3.0:1788201707.112488:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.112491:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.112522:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.112525:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.112541:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487297024, count: 4096 00000080:00200000:3.0:1788201707.112551:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487297024, bytes: 4096 00000080:00200000:3.0:1788201707.112554:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.112557:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.112559:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487297024, 487301120) 0 0 00000080:00200000:3.0:1788201707.112561:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.112566:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118969, 118969] 00020000:00200000:3.0:1788201707.112569:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487297024 chunk: [487297024, 487301120) 487301120, 4096 00020000:00200000:3.0:1788201707.112571:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.112583:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118969, 118969] 00000008:00010000:3.0:1788201707.112589:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.112591:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.112598:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112607:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487297024 487301119) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112616:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112624:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112635:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.112643:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487297024, 487301120) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.112648:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118969 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.112654:0:17463:0:(rw26.c:898:ll_write_end()) pos 487297024, len 4096, copied 4096 00000080:00200000:3.0:1788201707.112658:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c1799003[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118969] 00000080:00200000:3.0:1788201707.112660:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f03ffe90 fffffc0001029 2:0 ffff93c0c4402460 118969 lru 00000080:00200000:3.0:1788201707.112666:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bd00ac37 118969: 1< 2 - - > 2< 487297024 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70554 - -> 00000080:00200000:3.0:1788201707.112667:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c1799003 00000080:00200000:3.0:1788201707.112668:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.112670:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.112679:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.112691:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.112694:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.112696:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.112705:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.112708:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.112710:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.112714:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112722:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.112735:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.112738:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.112742:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.112745:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118969, 118969] 00000080:00200000:3.0:1788201707.112757:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.112760:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.112789:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.112792:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.112808:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487301120, count: 4096 00000080:00200000:3.0:1788201707.112818:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487301120, bytes: 4096 00000080:00200000:3.0:1788201707.112821:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.112825:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.112826:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487301120, 487305216) 0 0 00000080:00200000:3.0:1788201707.112845:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.112852:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118970, 118970] 00020000:00200000:3.0:1788201707.112855:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487301120 chunk: [487301120, 487305216) 487305216, 4096 00020000:00200000:3.0:1788201707.112857:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.112870:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118970, 118970] 00000008:00010000:3.0:1788201707.112875:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.112878:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.112885:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112893:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487301120 487305215) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112902:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112914:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.112921:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.112928:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487301120, 487305216) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.112934:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118970 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.112939:0:17463:0:(rw26.c:898:ll_write_end()) pos 487301120, len 4096, copied 4096 00000080:00200000:3.0:1788201707.112943:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005f715c0e[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118970] 00000080:00200000:3.0:1788201707.112946:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004553922f fffffc0001029 2:0 ffff93c0c44020e0 118970 lru 00000080:00200000:3.0:1788201707.112951:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000040470fbe 118970: 1< 2 - - > 2< 487301120 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70555 - -> 00000080:00200000:3.0:1788201707.112954:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005f715c0e 00000080:00200000:3.0:1788201707.112955:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.112956:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.112965:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.112977:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.112980:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.112982:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.112991:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.112994:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.112996:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.113000:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113026:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.113034:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.113037:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.113041:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.113044:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118970, 118970] 00000080:00200000:3.0:1788201707.113055:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.113058:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.113088:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.113092:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.113108:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487305216, count: 4096 00000080:00200000:3.0:1788201707.113118:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487305216, bytes: 4096 00000080:00200000:3.0:1788201707.113121:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.113124:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.113126:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487305216, 487309312) 0 0 00000080:00200000:3.0:1788201707.113128:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.113133:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118971, 118971] 00020000:00200000:3.0:1788201707.113136:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487305216 chunk: [487305216, 487309312) 487309312, 4096 00020000:00200000:3.0:1788201707.113138:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.113149:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118971, 118971] 00000008:00010000:3.0:1788201707.113153:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.113156:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.113163:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113172:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487305216 487309311) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113186:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113194:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113202:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.113209:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487305216, 487309312) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.113212:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118971 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.113217:0:17463:0:(rw26.c:898:ll_write_end()) pos 487305216, len 4096, copied 4096 00000080:00200000:3.0:1788201707.113221:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004970b0a8[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118971] 00000080:00200000:3.0:1788201707.113224:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004bf3390e fffffc0001029 2:0 ffff93c0c4402700 118971 lru 00000080:00200000:3.0:1788201707.113229:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000309b655d 118971: 1< 2 - - > 2< 487305216 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70556 - -> 00000080:00200000:3.0:1788201707.113231:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004970b0a8 00000080:00200000:3.0:1788201707.113232:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.113234:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.113242:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.113253:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.113256:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.113258:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.113268:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.113271:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.113274:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.113279:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113288:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.113295:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.113298:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.113301:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.113304:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118971, 118971] 00000080:00200000:3.0:1788201707.113314:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.113317:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.113343:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.113346:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.113361:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487309312, count: 4096 00000080:00200000:3.0:1788201707.113371:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487309312, bytes: 4096 00000080:00200000:3.0:1788201707.113374:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.113377:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.113379:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487309312, 487313408) 0 0 00000080:00200000:3.0:1788201707.113381:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.113386:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118972, 118972] 00020000:00200000:3.0:1788201707.113389:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487309312 chunk: [487309312, 487313408) 487313408, 4096 00020000:00200000:3.0:1788201707.113391:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.113403:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118972, 118972] 00000008:00010000:3.0:1788201707.113409:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.113411:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.113419:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113432:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487309312 487313407) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113442:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113449:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113456:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.113463:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487309312, 487313408) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.113468:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118972 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.113473:0:17463:0:(rw26.c:898:ll_write_end()) pos 487309312, len 4096, copied 4096 00000080:00200000:3.0:1788201707.113477:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000cbf26158[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118972] 00000080:00200000:3.0:1788201707.113480:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000404e7a6a fffffc0001029 2:0 ffff93c0c4402000 118972 lru 00000080:00200000:3.0:1788201707.113485:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000762b56c9 118972: 1< 2 - - > 2< 487309312 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70557 - -> 00000080:00200000:3.0:1788201707.113487:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000cbf26158 00000080:00200000:3.0:1788201707.113489:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.113490:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.113499:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.113511:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.113514:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.113516:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.113525:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.113527:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.113530:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.113534:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113542:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.113550:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.113553:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.113556:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.113559:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118972, 118972] 00000080:00200000:3.0:1788201707.113571:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.113574:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.113604:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.113608:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.113623:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487313408, count: 4096 00000080:00200000:3.0:1788201707.113634:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487313408, bytes: 4096 00000080:00200000:3.0:1788201707.113637:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.113640:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.113642:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487313408, 487317504) 0 0 00000080:00200000:3.0:1788201707.113645:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.113650:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118973, 118973] 00020000:00200000:3.0:1788201707.113653:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487313408 chunk: [487313408, 487317504) 487317504, 4096 00020000:00200000:3.0:1788201707.113655:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.113668:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118973, 118973] 00000008:00010000:3.0:1788201707.113673:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.113675:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.113683:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113692:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487313408 487317503) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113701:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113708:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113715:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.113723:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487313408, 487317504) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.113729:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118973 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.113734:0:17463:0:(rw26.c:898:ll_write_end()) pos 487313408, len 4096, copied 4096 00000080:00200000:3.0:1788201707.113738:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f6a08027[3 0000000075f4bf38 2 1 00000000783f07b1 0 0 118973] 00000080:00200000:3.0:1788201707.113740:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fc314ac9 fffffc0001029 2:0 ffff93c0c4402a80 118973 lru 00000080:00200000:3.0:1788201707.113746:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ec54b0c6 118973: 1< 2 - - > 2< 487313408 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70558 - -> 00000080:00200000:3.0:1788201707.113748:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f6a08027 00000080:00200000:3.0:1788201707.113749:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.113751:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.113759:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.113770:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.113773:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.113775:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.113784:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.113786:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.113789:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.113793:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.113800:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.113808:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e34e8 (ffff93c1d864f5a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.113810:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f5a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.113814:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.113817:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118973, 118973] 00000080:00200000:3.0:1788201707.113828:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.113854:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.113924:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.113930:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.113955:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487317504, count: 4096 00000080:00200000:2.0:1788201707.113973:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487317504, bytes: 4096 00000080:00200000:2.0:1788201707.113976:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.113980:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.113981:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487317504, 487321600) 0 0 00000080:00200000:2.0:1788201707.113984:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.113989:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118974, 118974] 00020000:00200000:2.0:1788201707.113993:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487317504 chunk: [487317504, 487321600) 487321600, 4096 00020000:00200000:2.0:1788201707.113995:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.114022:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118974, 118974] 00000008:00010000:2.0:1788201707.114029:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.114031:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.114040:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114050:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487317504 487321599) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114059:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114068:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114075:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.114082:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487317504, 487321600) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.114087:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118974 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.114093:0:17463:0:(rw26.c:898:ll_write_end()) pos 487317504, len 4096, copied 4096 00000080:00200000:2.0:1788201707.114098:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007b7ef576[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118974] 00000080:00200000:2.0:1788201707.114101:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e74e5013 fffffc0001029 2:0 ffff93c0c4402380 118974 lru 00000080:00200000:2.0:1788201707.114107:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000099dcab2d 118974: 1< 2 - - > 2< 487317504 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70559 - -> 00000080:00200000:2.0:1788201707.114111:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007b7ef576 00000080:00200000:2.0:1788201707.114111:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.114114:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.114123:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.114136:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.114139:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.114141:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.114153:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.114156:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.114158:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.114162:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114170:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.114179:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.114181:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.114187:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.114191:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118974, 118974] 00000080:00200000:2.0:1788201707.114204:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.114206:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.114229:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.114231:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.114243:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487321600, count: 4096 00000080:00200000:2.0:1788201707.114250:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487321600, bytes: 4096 00000080:00200000:2.0:1788201707.114252:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.114255:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.114256:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487321600, 487325696) 0 0 00000080:00200000:2.0:1788201707.114258:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.114262:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118975, 118975] 00020000:00200000:2.0:1788201707.114264:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487321600 chunk: [487321600, 487325696) 487325696, 4096 00020000:00200000:2.0:1788201707.114265:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.114272:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118975, 118975] 00000008:00010000:2.0:1788201707.114276:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.114277:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.114283:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114289:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487321600 487325695) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114295:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114299:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114303:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.114308:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487321600, 487325696) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.114311:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118975 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.114316:0:17463:0:(rw26.c:898:ll_write_end()) pos 487321600, len 4096, copied 4096 00000080:00200000:2.0:1788201707.114318:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a40b9fcb[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118975] 00000080:00200000:2.0:1788201707.114321:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000ccb93359 fffffc0001029 2:0 ffff93c0c44028c0 118975 lru 00000080:00200000:2.0:1788201707.114325:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000098edb586 118975: 1< 2 - - > 2< 487321600 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70560 - -> 00000080:00200000:2.0:1788201707.114327:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a40b9fcb 00000080:00200000:2.0:1788201707.114327:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.114329:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.114344:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.114353:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.114355:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.114356:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.114364:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.114366:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.114367:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.114370:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114375:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.114380:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.114382:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.114384:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.114386:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118975, 118975] 00000080:00200000:2.0:1788201707.114393:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.114395:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.114414:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.114417:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.114427:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487325696, count: 4096 00000080:00200000:2.0:1788201707.114439:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487325696, bytes: 4096 00000080:00200000:2.0:1788201707.114441:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.114443:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.114444:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487325696, 487329792) 0 0 00000080:00200000:2.0:1788201707.114446:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.114450:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118976, 118976] 00020000:00200000:2.0:1788201707.114452:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487325696 chunk: [487325696, 487329792) 487329792, 4096 00020000:00200000:2.0:1788201707.114453:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.114460:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118976, 118976] 00000008:00010000:2.0:1788201707.114463:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.114465:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.114470:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114476:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487325696 487329791) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114482:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114486:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114490:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.114494:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487325696, 487329792) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.114497:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118976 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.114501:0:17463:0:(rw26.c:898:ll_write_end()) pos 487325696, len 4096, copied 4096 00000080:00200000:2.0:1788201707.114504:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000593b17ed[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118976] 00000080:00200000:2.0:1788201707.114506:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a14a8393 fffffc0001029 2:0 ffff93c0c4402ee0 118976 lru 00000080:00200000:2.0:1788201707.114510:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d7319443 118976: 1< 2 - - > 2< 487325696 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70561 - -> 00000080:00200000:2.0:1788201707.114511:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000593b17ed 00000080:00200000:2.0:1788201707.114512:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.114514:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.114522:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.114534:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.114537:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.114539:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.114549:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.114552:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.114554:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.114558:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114566:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.114574:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.114577:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.114581:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.114584:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118976, 118976] 00000080:00200000:2.0:1788201707.114595:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.114599:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.114619:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.114622:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.114633:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487329792, count: 4096 00000080:00200000:2.0:1788201707.114639:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487329792, bytes: 4096 00000080:00200000:2.0:1788201707.114641:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.114644:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.114645:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487329792, 487333888) 0 0 00000080:00200000:2.0:1788201707.114646:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.114650:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118977, 118977] 00020000:00200000:2.0:1788201707.114652:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487329792 chunk: [487329792, 487333888) 487333888, 4096 00020000:00200000:2.0:1788201707.114654:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.114661:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118977, 118977] 00000008:00010000:2.0:1788201707.114664:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.114665:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.114671:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114677:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487329792 487333887) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114683:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114687:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114691:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.114695:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487329792, 487333888) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.114699:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118977 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.114703:0:17463:0:(rw26.c:898:ll_write_end()) pos 487329792, len 4096, copied 4096 00000080:00200000:2.0:1788201707.114706:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c28ff26a[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118977] 00000080:00200000:2.0:1788201707.114708:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bf21421f fffffc0001029 2:0 ffff93c0c44027e0 118977 lru 00000080:00200000:2.0:1788201707.114712:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000fddc2010 118977: 1< 2 - - > 2< 487329792 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70562 - -> 00000080:00200000:2.0:1788201707.114714:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c28ff26a 00000080:00200000:2.0:1788201707.114715:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.114716:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.114722:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.114730:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.114732:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.114733:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.114739:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.114741:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.114743:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.114745:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114750:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.114755:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.114756:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.114758:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.114760:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118977, 118977] 00000080:00200000:2.0:1788201707.114767:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.114769:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.114794:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.114796:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.114807:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487333888, count: 4096 00000080:00200000:2.0:1788201707.114813:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487333888, bytes: 4096 00000080:00200000:2.0:1788201707.114815:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.114817:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.114818:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487333888, 487337984) 0 0 00000080:00200000:2.0:1788201707.114820:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.114823:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118978, 118978] 00020000:00200000:2.0:1788201707.114825:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487333888 chunk: [487333888, 487337984) 487337984, 4096 00020000:00200000:2.0:1788201707.114827:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.114848:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118978, 118978] 00000008:00010000:2.0:1788201707.114851:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.114853:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.114858:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114864:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487333888 487337983) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114870:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114874:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114878:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.114886:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487333888, 487337984) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.114890:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118978 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.114893:0:17463:0:(rw26.c:898:ll_write_end()) pos 487333888, len 4096, copied 4096 00000080:00200000:2.0:1788201707.114896:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008f98af5d[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118978] 00000080:00200000:2.0:1788201707.114898:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008513b3ca fffffc0001029 2:0 ffff93c0c44022a0 118978 lru 00000080:00200000:2.0:1788201707.114902:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d40e6847 118978: 1< 2 - - > 2< 487333888 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70563 - -> 00000080:00200000:2.0:1788201707.114903:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008f98af5d 00000080:00200000:2.0:1788201707.114904:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.114905:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.114911:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.114918:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.114920:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.114921:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.114928:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.114929:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.114931:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.114934:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.114939:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.114944:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.114945:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.114948:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.114950:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118978, 118978] 00000080:00200000:2.0:1788201707.114956:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.114958:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.114976:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.114978:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.114988:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487337984, count: 4096 00000080:00200000:2.0:1788201707.114994:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487337984, bytes: 4096 00000080:00200000:2.0:1788201707.114996:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.114998:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.114999:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487337984, 487342080) 0 0 00000080:00200000:2.0:1788201707.115001:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.115004:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118979, 118979] 00020000:00200000:2.0:1788201707.115019:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487337984 chunk: [487337984, 487342080) 487342080, 4096 00020000:00200000:2.0:1788201707.115021:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.115032:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118979, 118979] 00000008:00010000:2.0:1788201707.115035:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.115037:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.115042:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115048:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487337984 487342079) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115053:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115058:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115066:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.115070:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487337984, 487342080) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.115073:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118979 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.115077:0:17463:0:(rw26.c:898:ll_write_end()) pos 487337984, len 4096, copied 4096 00000080:00200000:2.0:1788201707.115079:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000072fd7032[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118979] 00000080:00200000:2.0:1788201707.115081:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000069c48709 fffffc0001029 2:0 ffff93c0c4402e00 118979 lru 00000080:00200000:2.0:1788201707.115085:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000077070ced 118979: 1< 2 - - > 2< 487337984 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70564 - -> 00000080:00200000:2.0:1788201707.115086:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000072fd7032 00000080:00200000:2.0:1788201707.115087:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.115088:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.115094:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.115102:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.115104:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.115105:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.115111:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.115113:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.115114:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.115117:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115122:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.115131:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.115132:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.115135:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.115137:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118979, 118979] 00000080:00200000:2.0:1788201707.115143:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.115145:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.115163:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.115165:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.115176:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487342080, count: 4096 00000080:00200000:2.0:1788201707.115182:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487342080, bytes: 4096 00000080:00200000:2.0:1788201707.115183:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.115186:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.115187:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487342080, 487346176) 0 0 00000080:00200000:2.0:1788201707.115188:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.115192:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118980, 118980] 00020000:00200000:2.0:1788201707.115194:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487342080 chunk: [487342080, 487346176) 487346176, 4096 00020000:00200000:2.0:1788201707.115195:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.115201:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118980, 118980] 00000008:00010000:2.0:1788201707.115205:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.115206:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.115211:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115217:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487342080 487346175) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115226:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115238:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115246:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.115254:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487342080, 487346176) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.115259:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118980 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.115264:0:17463:0:(rw26.c:898:ll_write_end()) pos 487342080, len 4096, copied 4096 00000080:00200000:2.0:1788201707.115268:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000b6fd3357[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118980] 00000080:00200000:2.0:1788201707.115271:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003efe3734 fffffc0001029 2:0 ffff93c0c44021c0 118980 lru 00000080:00200000:2.0:1788201707.115275:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000000863ebe 118980: 1< 2 - - > 2< 487342080 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70565 - -> 00000080:00200000:2.0:1788201707.115277:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000b6fd3357 00000080:00200000:2.0:1788201707.115278:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.115279:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.115286:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.115294:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.115296:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.115297:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.115304:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.115305:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.115307:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.115310:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115318:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.115323:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.115324:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.115327:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.115329:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118980, 118980] 00000080:00200000:2.0:1788201707.115336:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.115338:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.115356:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.115358:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.115368:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487346176, count: 4096 00000080:00200000:2.0:1788201707.115374:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487346176, bytes: 4096 00000080:00200000:2.0:1788201707.115376:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.115379:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.115379:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487346176, 487350272) 0 0 00000080:00200000:2.0:1788201707.115381:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.115385:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118981, 118981] 00020000:00200000:2.0:1788201707.115387:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487346176 chunk: [487346176, 487350272) 487350272, 4096 00020000:00200000:2.0:1788201707.115388:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.115395:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118981, 118981] 00000008:00010000:2.0:1788201707.115398:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.115400:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.115405:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115410:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487346176 487350271) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115420:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115424:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115428:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.115432:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487346176, 487350272) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.115435:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118981 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.115438:0:17463:0:(rw26.c:898:ll_write_end()) pos 487346176, len 4096, copied 4096 00000080:00200000:2.0:1788201707.115441:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005eeae5ab[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118981] 00000080:00200000:2.0:1788201707.115442:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a717a1b6 fffffc0001029 2:0 ffff93c0c4402620 118981 lru 00000080:00200000:2.0:1788201707.115446:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000152de654 118981: 1< 2 - - > 2< 487346176 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70566 - -> 00000080:00200000:2.0:1788201707.115447:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005eeae5ab 00000080:00200000:2.0:1788201707.115448:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.115449:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.115454:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.115461:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.115463:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.115464:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.115470:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.115472:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.115474:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.115476:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115481:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.115485:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.115487:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.115490:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.115492:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118981, 118981] 00000080:00200000:2.0:1788201707.115498:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.115500:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.115520:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.115523:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.115538:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487350272, count: 4096 00000080:00200000:2.0:1788201707.115548:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487350272, bytes: 4096 00000080:00200000:2.0:1788201707.115551:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.115554:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.115556:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487350272, 487354368) 0 0 00000080:00200000:2.0:1788201707.115558:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.115563:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118982, 118982] 00020000:00200000:2.0:1788201707.115566:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487350272 chunk: [487350272, 487354368) 487354368, 4096 00020000:00200000:2.0:1788201707.115568:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.115580:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118982, 118982] 00000008:00010000:2.0:1788201707.115585:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.115588:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.115593:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115603:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487350272 487354367) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115608:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115613:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115617:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.115621:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487350272, 487354368) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.115625:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118982 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.115628:0:17463:0:(rw26.c:898:ll_write_end()) pos 487350272, len 4096, copied 4096 00000080:00200000:2.0:1788201707.115631:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d1a8a7d9[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118982] 00000080:00200000:2.0:1788201707.115632:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000168cc77b fffffc0001029 2:0 ffff93c0c4402d20 118982 lru 00000080:00200000:2.0:1788201707.115636:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000232a8bbc 118982: 1< 2 - - > 2< 487350272 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70567 - -> 00000080:00200000:2.0:1788201707.115637:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d1a8a7d9 00000080:00200000:2.0:1788201707.115638:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.115639:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.115645:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.115653:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.115654:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.115656:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.115663:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.115664:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.115666:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.115668:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115673:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.115678:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.115679:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.115682:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.115684:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118982, 118982] 00000080:00200000:2.0:1788201707.115691:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.115693:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.115711:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.115713:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.115723:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487354368, count: 4096 00000080:00200000:2.0:1788201707.115729:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487354368, bytes: 4096 00000080:00200000:2.0:1788201707.115731:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.115733:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.115734:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487354368, 487358464) 0 0 00000080:00200000:2.0:1788201707.115736:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.115739:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118983, 118983] 00020000:00200000:2.0:1788201707.115741:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487354368 chunk: [487354368, 487358464) 487358464, 4096 00020000:00200000:2.0:1788201707.115743:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.115749:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118983, 118983] 00000008:00010000:2.0:1788201707.115752:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.115753:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.115759:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115764:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487354368 487358463) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115770:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115774:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115778:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.115782:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487354368, 487358464) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.115785:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118983 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.115788:0:17463:0:(rw26.c:898:ll_write_end()) pos 487354368, len 4096, copied 4096 00000080:00200000:2.0:1788201707.115791:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002bfcba35[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118983] 00000080:00200000:2.0:1788201707.115793:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000008a2a91b7 fffffc0001029 2:0 ffff93c0c44029a0 118983 lru 00000080:00200000:2.0:1788201707.115796:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000556f4253 118983: 1< 2 - - > 2< 487354368 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70568 - -> 00000080:00200000:2.0:1788201707.115798:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002bfcba35 00000080:00200000:2.0:1788201707.115799:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.115800:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.115805:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.115812:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.115814:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.115816:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.115821:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.115823:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.115825:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.115827:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115853:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.115859:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.115861:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.115864:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.115867:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118983, 118983] 00000080:00200000:2.0:1788201707.115876:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.115878:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.115896:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.115899:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.115909:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487358464, count: 4096 00000080:00200000:2.0:1788201707.115915:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487358464, bytes: 4096 00000080:00200000:2.0:1788201707.115917:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.115919:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.115920:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487358464, 487362560) 0 0 00000080:00200000:2.0:1788201707.115921:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.115925:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118984, 118984] 00020000:00200000:2.0:1788201707.115926:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487358464 chunk: [487358464, 487362560) 487362560, 4096 00020000:00200000:2.0:1788201707.115928:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.115934:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118984, 118984] 00000008:00010000:2.0:1788201707.115938:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.115939:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.115944:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115950:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487358464 487362559) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115955:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115960:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.115963:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.115968:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487358464, 487362560) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.115971:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118984 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.115975:0:17463:0:(rw26.c:898:ll_write_end()) pos 487358464, len 4096, copied 4096 00000080:00200000:2.0:1788201707.115977:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c718679b[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118984] 00000080:00200000:2.0:1788201707.115979:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000abad9644 fffffc0001029 2:0 ffff93c0c4402b60 118984 lru 00000080:00200000:2.0:1788201707.115982:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000c71dfcc 118984: 1< 2 - - > 2< 487358464 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70569 - -> 00000080:00200000:2.0:1788201707.115984:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c718679b 00000080:00200000:2.0:1788201707.115984:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.115986:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.115991:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.116002:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.116004:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.116005:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.116027:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.116029:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.116030:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.116033:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116039:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.116044:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.116045:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.116047:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.116049:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118984, 118984] 00000080:00200000:2.0:1788201707.116056:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.116058:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.116077:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.116079:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.116089:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487362560, count: 4096 00000080:00200000:2.0:1788201707.116095:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487362560, bytes: 4096 00000080:00200000:2.0:1788201707.116097:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.116099:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.116100:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487362560, 487366656) 0 0 00000080:00200000:2.0:1788201707.116102:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.116105:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118985, 118985] 00020000:00200000:2.0:1788201707.116107:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487362560 chunk: [487362560, 487366656) 487366656, 4096 00020000:00200000:2.0:1788201707.116109:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.116116:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118985, 118985] 00000008:00010000:2.0:1788201707.116119:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.116120:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.116125:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116131:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487362560 487366655) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116136:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116141:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116145:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.116149:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487362560, 487366656) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.116151:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118985 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.116155:0:17463:0:(rw26.c:898:ll_write_end()) pos 487362560, len 4096, copied 4096 00000080:00200000:2.0:1788201707.116157:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000029a8068a[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118985] 00000080:00200000:2.0:1788201707.116159:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000021225a64 fffffc0001029 2:0 ffff93c0c44030e0 118985 lru 00000080:00200000:2.0:1788201707.116162:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000009e9c000a 118985: 1< 2 - - > 2< 487362560 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70570 - -> 00000080:00200000:2.0:1788201707.116164:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000029a8068a 00000080:00200000:2.0:1788201707.116165:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.116166:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.116172:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.116179:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.116181:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.116182:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.116188:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.116189:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.116191:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.116194:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116198:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.116203:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.116204:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.116206:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.116208:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118985, 118985] 00000080:00200000:2.0:1788201707.116217:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.116220:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.116246:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.116250:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.116265:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487366656, count: 4096 00000080:00200000:2.0:1788201707.116274:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487366656, bytes: 4096 00000080:00200000:2.0:1788201707.116276:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.116278:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.116279:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487366656, 487370752) 0 0 00000080:00200000:2.0:1788201707.116281:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.116284:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118986, 118986] 00020000:00200000:2.0:1788201707.116287:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487366656 chunk: [487366656, 487370752) 487370752, 4096 00020000:00200000:2.0:1788201707.116288:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.116295:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118986, 118986] 00000008:00010000:2.0:1788201707.116298:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.116300:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.116305:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116311:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487366656 487370751) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116316:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116321:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116325:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.116330:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487366656, 487370752) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.116333:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118986 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.116336:0:17463:0:(rw26.c:898:ll_write_end()) pos 487366656, len 4096, copied 4096 00000080:00200000:2.0:1788201707.116338:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003a2b3cb0[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118986] 00000080:00200000:2.0:1788201707.116340:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000dffbcb65 fffffc0001029 2:0 ffff93c0c4403700 118986 lru 00000080:00200000:2.0:1788201707.116344:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000872c535 118986: 1< 2 - - > 2< 487366656 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70571 - -> 00000080:00200000:2.0:1788201707.116346:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003a2b3cb0 00000080:00200000:2.0:1788201707.116347:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.116348:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.116354:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.116361:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.116363:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.116364:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.116370:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.116372:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.116374:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.116376:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116381:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.116385:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.116387:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.116389:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.116391:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118986, 118986] 00000080:00200000:2.0:1788201707.116398:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.116400:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.116418:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.116420:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.116430:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487370752, count: 4096 00000080:00200000:2.0:1788201707.116436:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487370752, bytes: 4096 00000080:00200000:2.0:1788201707.116438:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.116440:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.116441:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487370752, 487374848) 0 0 00000080:00200000:2.0:1788201707.116443:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.116446:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118987, 118987] 00020000:00200000:2.0:1788201707.116448:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487370752 chunk: [487370752, 487374848) 487374848, 4096 00020000:00200000:2.0:1788201707.116449:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.116456:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118987, 118987] 00000008:00010000:2.0:1788201707.116459:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.116460:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.116465:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116471:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487370752 487374847) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116476:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116481:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116485:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.116489:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487370752, 487374848) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.116492:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118987 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.116495:0:17463:0:(rw26.c:898:ll_write_end()) pos 487370752, len 4096, copied 4096 00000080:00200000:2.0:1788201707.116497:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000065271e32[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118987] 00000080:00200000:2.0:1788201707.116499:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a5971288 fffffc0001029 2:0 ffff93c0c4403000 118987 lru 00000080:00200000:2.0:1788201707.116503:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000081d5f4e 118987: 1< 2 - - > 2< 487370752 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70572 - -> 00000080:00200000:2.0:1788201707.116505:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000065271e32 00000080:00200000:2.0:1788201707.116505:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.116507:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.116512:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.116522:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.116525:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.116527:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.116537:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.116539:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.116542:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.116546:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116554:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.116562:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.116565:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.116569:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.116572:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118987, 118987] 00000080:00200000:2.0:1788201707.116579:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.116581:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.116600:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.116603:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.116613:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487374848, count: 4096 00000080:00200000:2.0:1788201707.116619:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487374848, bytes: 4096 00000080:00200000:2.0:1788201707.116621:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.116623:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.116624:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487374848, 487378944) 0 0 00000080:00200000:2.0:1788201707.116626:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.116629:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118988, 118988] 00020000:00200000:2.0:1788201707.116631:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487374848 chunk: [487374848, 487378944) 487378944, 4096 00020000:00200000:2.0:1788201707.116632:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.116639:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118988, 118988] 00000008:00010000:2.0:1788201707.116642:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.116643:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.116648:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116655:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487374848 487378943) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116660:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116665:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116669:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.116674:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487374848, 487378944) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.116677:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118988 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.116681:0:17463:0:(rw26.c:898:ll_write_end()) pos 487374848, len 4096, copied 4096 00000080:00200000:2.0:1788201707.116683:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001af9685b[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118988] 00000080:00200000:2.0:1788201707.116685:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000049c78c59 fffffc0001029 2:0 ffff93c0c4403a80 118988 lru 00000080:00200000:2.0:1788201707.116689:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f0157bca 118988: 1< 2 - - > 2< 487374848 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70573 - -> 00000080:00200000:2.0:1788201707.116690:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001af9685b 00000080:00200000:2.0:1788201707.116691:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.116692:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.116698:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.116705:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.116707:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.116708:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.116714:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.116716:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.116717:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.116720:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116724:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.116729:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.116731:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.116733:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.116735:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118988, 118988] 00000080:00200000:2.0:1788201707.116742:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.116744:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.116761:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.116764:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.116774:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487378944, count: 4096 00000080:00200000:2.0:1788201707.116780:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487378944, bytes: 4096 00000080:00200000:2.0:1788201707.116781:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.116784:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.116785:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487378944, 487383040) 0 0 00000080:00200000:2.0:1788201707.116786:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.116789:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118989, 118989] 00020000:00200000:2.0:1788201707.116791:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487378944 chunk: [487378944, 487383040) 487383040, 4096 00020000:00200000:2.0:1788201707.116793:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.116799:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118989, 118989] 00000008:00010000:2.0:1788201707.116802:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.116803:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.116808:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116814:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487378944 487383039) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116820:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116824:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116828:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.116846:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487378944, 487383040) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.116850:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118989 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.116854:0:17463:0:(rw26.c:898:ll_write_end()) pos 487378944, len 4096, copied 4096 00000080:00200000:2.0:1788201707.116856:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c5b6655c[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118989] 00000080:00200000:2.0:1788201707.116858:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000ef14531 fffffc0001029 2:0 ffff93c0c4403380 118989 lru 00000080:00200000:2.0:1788201707.116862:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bf1856ca 118989: 1< 2 - - > 2< 487378944 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70574 - -> 00000080:00200000:2.0:1788201707.116863:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c5b6655c 00000080:00200000:2.0:1788201707.116864:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.116865:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.116871:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.116878:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.116880:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.116881:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.116887:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.116889:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.116891:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.116893:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116898:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.116902:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.116904:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.116906:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.116908:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118989, 118989] 00000080:00200000:2.0:1788201707.116915:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.116916:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.116934:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.116936:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.116946:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487383040, count: 4096 00000080:00200000:2.0:1788201707.116952:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487383040, bytes: 4096 00000080:00200000:2.0:1788201707.116954:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.116957:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.116958:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487383040, 487387136) 0 0 00000080:00200000:2.0:1788201707.116959:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.116962:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118990, 118990] 00020000:00200000:2.0:1788201707.116964:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487383040 chunk: [487383040, 487387136) 487387136, 4096 00020000:00200000:2.0:1788201707.116966:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.116972:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118990, 118990] 00000008:00010000:2.0:1788201707.116975:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.116976:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.116981:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116987:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487383040 487387135) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116992:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.116997:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117001:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.117022:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487383040, 487387136) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.117025:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118990 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.117029:0:17463:0:(rw26.c:898:ll_write_end()) pos 487383040, len 4096, copied 4096 00000080:00200000:2.0:1788201707.117031:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000006e65e75[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118990] 00000080:00200000:2.0:1788201707.117034:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000004947ff95 fffffc0001029 2:0 ffff93c0c44038c0 118990 lru 00000080:00200000:2.0:1788201707.117037:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000059ed8010 118990: 1< 2 - - > 2< 487383040 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70575 - -> 00000080:00200000:2.0:1788201707.117039:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000006e65e75 00000080:00200000:2.0:1788201707.117039:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.117041:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.117047:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.117055:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.117056:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.117058:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.117064:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.117066:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.117067:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.117070:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117074:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.117080:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.117081:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.117090:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.117092:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118990, 118990] 00000080:00200000:2.0:1788201707.117100:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.117102:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.117120:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.117122:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.117132:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487387136, count: 4096 00000080:00200000:2.0:1788201707.117138:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487387136, bytes: 4096 00000080:00200000:2.0:1788201707.117140:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.117142:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.117143:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487387136, 487391232) 0 0 00000080:00200000:2.0:1788201707.117145:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.117148:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118991, 118991] 00020000:00200000:2.0:1788201707.117150:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487387136 chunk: [487387136, 487391232) 487391232, 4096 00020000:00200000:2.0:1788201707.117151:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.117158:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118991, 118991] 00000008:00010000:2.0:1788201707.117161:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.117162:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.117168:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117173:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487387136 487391231) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117178:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117183:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117190:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.117195:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487387136, 487391232) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.117198:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118991 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.117201:0:17463:0:(rw26.c:898:ll_write_end()) pos 487387136, len 4096, copied 4096 00000080:00200000:2.0:1788201707.117203:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c6f47680[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118991] 00000080:00200000:2.0:1788201707.117205:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000001fc396c fffffc0001029 2:0 ffff93c0c4403ee0 118991 lru 00000080:00200000:2.0:1788201707.117209:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001f7ab829 118991: 1< 2 - - > 2< 487387136 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70576 - -> 00000080:00200000:2.0:1788201707.117211:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c6f47680 00000080:00200000:2.0:1788201707.117212:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.117213:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.117221:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.117233:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.117235:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.117238:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.117246:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.117249:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.117251:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.117256:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117264:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.117274:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.117276:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.117279:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.117281:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118991, 118991] 00000080:00200000:2.0:1788201707.117289:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.117291:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.117309:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.117312:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.117322:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487391232, count: 4096 00000080:00200000:2.0:1788201707.117328:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487391232, bytes: 4096 00000080:00200000:2.0:1788201707.117330:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.117332:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.117333:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487391232, 487395328) 0 0 00000080:00200000:2.0:1788201707.117335:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.117338:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118992, 118992] 00020000:00200000:2.0:1788201707.117340:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487391232 chunk: [487391232, 487395328) 487395328, 4096 00020000:00200000:2.0:1788201707.117341:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.117348:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118992, 118992] 00000008:00010000:2.0:1788201707.117351:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.117352:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.117358:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117363:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487391232 487395327) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117369:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117377:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117381:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.117385:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487391232, 487395328) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.117388:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118992 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.117392:0:17463:0:(rw26.c:898:ll_write_end()) pos 487391232, len 4096, copied 4096 00000080:00200000:2.0:1788201707.117394:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000955aa6c5[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118992] 00000080:00200000:2.0:1788201707.117396:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d2da3065 fffffc0001029 2:0 ffff93c0c44037e0 118992 lru 00000080:00200000:2.0:1788201707.117399:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e61e6d2a 118992: 1< 2 - - > 2< 487391232 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70577 - -> 00000080:00200000:2.0:1788201707.117400:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000955aa6c5 00000080:00200000:2.0:1788201707.117401:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.117402:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.117408:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.117415:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.117417:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.117418:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.117424:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.117426:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.117427:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.117430:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117438:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.117443:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.117444:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.117447:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.117449:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118992, 118992] 00000080:00200000:2.0:1788201707.117455:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.117457:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.117475:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.117477:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.117487:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487395328, count: 4096 00000080:00200000:2.0:1788201707.117493:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487395328, bytes: 4096 00000080:00200000:2.0:1788201707.117495:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.117497:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.117498:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487395328, 487399424) 0 0 00000080:00200000:2.0:1788201707.117499:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.117502:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118993, 118993] 00020000:00200000:2.0:1788201707.117504:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487395328 chunk: [487395328, 487399424) 487399424, 4096 00020000:00200000:2.0:1788201707.117506:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.117512:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118993, 118993] 00000008:00010000:2.0:1788201707.117516:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.117518:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.117526:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117535:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487395328 487399423) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117547:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117555:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117561:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.117569:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487395328, 487399424) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.117574:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118993 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.117578:0:17463:0:(rw26.c:898:ll_write_end()) pos 487395328, len 4096, copied 4096 00000080:00200000:2.0:1788201707.117580:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000c91a675[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118993] 00000080:00200000:2.0:1788201707.117582:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009d8710d9 fffffc0001029 2:0 ffff93c0c44032a0 118993 lru 00000080:00200000:2.0:1788201707.117586:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000648450eb 118993: 1< 2 - - > 2< 487395328 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70578 - -> 00000080:00200000:2.0:1788201707.117587:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000c91a675 00000080:00200000:2.0:1788201707.117588:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.117590:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.117596:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.117603:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.117605:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.117606:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.117613:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.117615:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.117617:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.117619:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117624:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.117629:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.117630:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.117633:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.117635:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118993, 118993] 00000080:00200000:2.0:1788201707.117642:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.117644:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.117662:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.117664:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.117674:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487399424, count: 4096 00000080:00200000:2.0:1788201707.117680:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487399424, bytes: 4096 00000080:00200000:2.0:1788201707.117682:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.117684:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.117685:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487399424, 487403520) 0 0 00000080:00200000:2.0:1788201707.117687:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.117690:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118994, 118994] 00020000:00200000:2.0:1788201707.117692:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487399424 chunk: [487399424, 487403520) 487403520, 4096 00020000:00200000:2.0:1788201707.117693:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.117700:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118994, 118994] 00000008:00010000:2.0:1788201707.117703:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.117704:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.117709:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117718:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487399424 487403519) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117724:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117728:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117732:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.117736:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487399424, 487403520) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.117739:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118994 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.117742:0:17463:0:(rw26.c:898:ll_write_end()) pos 487399424, len 4096, copied 4096 00000080:00200000:2.0:1788201707.117744:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000b719749[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118994] 00000080:00200000:2.0:1788201707.117746:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e5f93898 fffffc0001029 2:0 ffff93c0c4403e00 118994 lru 00000080:00200000:2.0:1788201707.117750:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000036babfda 118994: 1< 2 - - > 2< 487399424 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70579 - -> 00000080:00200000:2.0:1788201707.117751:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000b719749 00000080:00200000:2.0:1788201707.117751:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.117752:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.117757:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.117765:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.117766:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.117768:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.117773:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.117775:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.117777:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.117779:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117784:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.117789:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.117791:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.117793:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.117795:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118994, 118994] 00000080:00200000:2.0:1788201707.117801:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.117803:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.117821:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.117823:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.117847:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487403520, count: 4096 00000080:00200000:2.0:1788201707.117854:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487403520, bytes: 4096 00000080:00200000:2.0:1788201707.117856:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.117858:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.117859:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487403520, 487407616) 0 0 00000080:00200000:2.0:1788201707.117860:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.117864:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118995, 118995] 00020000:00200000:2.0:1788201707.117865:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487403520 chunk: [487403520, 487407616) 487407616, 4096 00020000:00200000:2.0:1788201707.117867:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.117873:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118995, 118995] 00000008:00010000:2.0:1788201707.117876:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.117878:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.117883:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117889:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487403520 487407615) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117894:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117899:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117903:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.117907:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487403520, 487407616) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.117910:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118995 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.117913:0:17463:0:(rw26.c:898:ll_write_end()) pos 487403520, len 4096, copied 4096 00000080:00200000:2.0:1788201707.117916:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000916490db[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118995] 00000080:00200000:2.0:1788201707.117917:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a4584150 fffffc0001029 2:0 ffff93c0c44031c0 118995 lru 00000080:00200000:2.0:1788201707.117921:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008081b807 118995: 1< 2 - - > 2< 487403520 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70580 - -> 00000080:00200000:2.0:1788201707.117922:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000916490db 00000080:00200000:2.0:1788201707.117922:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.117924:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.117929:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.117937:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.117938:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.117940:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.117946:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.117948:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.117949:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.117951:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.117956:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.117961:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.117962:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.117965:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.117967:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118995, 118995] 00000080:00200000:2.0:1788201707.117973:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.117975:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.117993:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.117995:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.118005:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487407616, count: 4096 00000080:00200000:2.0:1788201707.118024:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487407616, bytes: 4096 00000080:00200000:2.0:1788201707.118026:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.118029:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.118030:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487407616, 487411712) 0 0 00000080:00200000:2.0:1788201707.118031:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.118035:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118996, 118996] 00020000:00200000:2.0:1788201707.118037:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487407616 chunk: [487407616, 487411712) 487411712, 4096 00020000:00200000:2.0:1788201707.118038:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.118045:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118996, 118996] 00000008:00010000:2.0:1788201707.118049:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.118050:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.118055:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118061:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487407616 487411711) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118067:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118071:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118075:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.118079:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487407616, 487411712) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.118083:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118996 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.118086:0:17463:0:(rw26.c:898:ll_write_end()) pos 487407616, len 4096, copied 4096 00000080:00200000:2.0:1788201707.118089:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006d1c88eb[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118996] 00000080:00200000:2.0:1788201707.118090:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005c1afcbf fffffc0001029 2:0 ffff93c0c4403620 118996 lru 00000080:00200000:2.0:1788201707.118094:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000085cc9a57 118996: 1< 2 - - > 2< 487407616 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70581 - -> 00000080:00200000:2.0:1788201707.118096:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006d1c88eb 00000080:00200000:2.0:1788201707.118096:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.118098:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.118103:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.118114:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.118115:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.118117:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.118123:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.118125:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.118126:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.118129:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118133:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.118138:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.118139:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.118142:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.118144:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118996, 118996] 00000080:00200000:2.0:1788201707.118150:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.118152:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.118170:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.118173:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.118182:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487411712, count: 4096 00000080:00200000:2.0:1788201707.118188:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487411712, bytes: 4096 00000080:00200000:2.0:1788201707.118190:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.118192:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.118193:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487411712, 487415808) 0 0 00000080:00200000:2.0:1788201707.118195:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.118198:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118997, 118997] 00020000:00200000:2.0:1788201707.118200:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487411712 chunk: [487411712, 487415808) 487415808, 4096 00020000:00200000:2.0:1788201707.118202:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.118208:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118997, 118997] 00000008:00010000:2.0:1788201707.118212:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.118214:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.118221:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118230:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487411712 487415807) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118238:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118246:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118254:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.118262:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487411712, 487415808) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.118268:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118997 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.118272:0:17463:0:(rw26.c:898:ll_write_end()) pos 487411712, len 4096, copied 4096 00000080:00200000:2.0:1788201707.118274:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000046063b4[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118997] 00000080:00200000:2.0:1788201707.118276:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000c132ec4 fffffc0001029 2:0 ffff93c0c4403d20 118997 lru 00000080:00200000:2.0:1788201707.118280:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000267a5260 118997: 1< 2 - - > 2< 487411712 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70582 - -> 00000080:00200000:2.0:1788201707.118282:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000046063b4 00000080:00200000:2.0:1788201707.118282:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.118284:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.118290:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.118298:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.118300:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.118301:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.118307:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.118309:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.118310:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.118313:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118318:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.118323:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.118324:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.118327:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.118329:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118997, 118997] 00000080:00200000:2.0:1788201707.118336:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.118338:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.118355:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.118358:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.118368:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487415808, count: 4096 00000080:00200000:2.0:1788201707.118374:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487415808, bytes: 4096 00000080:00200000:2.0:1788201707.118376:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.118378:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.118379:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487415808, 487419904) 0 0 00000080:00200000:2.0:1788201707.118381:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.118384:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118998, 118998] 00020000:00200000:2.0:1788201707.118386:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487415808 chunk: [487415808, 487419904) 487419904, 4096 00020000:00200000:2.0:1788201707.118388:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.118394:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118998, 118998] 00000008:00010000:2.0:1788201707.118397:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.118398:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.118403:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118409:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487415808 487419903) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118414:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118419:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118423:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.118427:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487415808, 487419904) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.118430:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118998 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.118434:0:17463:0:(rw26.c:898:ll_write_end()) pos 487415808, len 4096, copied 4096 00000080:00200000:2.0:1788201707.118436:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000a6106153[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118998] 00000080:00200000:2.0:1788201707.118438:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000b3472bcc fffffc0001029 2:0 ffff93c0c44039a0 118998 lru 00000080:00200000:2.0:1788201707.118442:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002ecad101 118998: 1< 2 - - > 2< 487415808 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70583 - -> 00000080:00200000:2.0:1788201707.118444:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000a6106153 00000080:00200000:2.0:1788201707.118445:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.118446:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.118451:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.118459:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.118461:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.118462:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.118467:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.118469:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.118471:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.118473:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118478:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.118482:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.118483:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.118486:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.118488:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118998, 118998] 00000080:00200000:2.0:1788201707.118494:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.118496:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.118514:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.118518:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.118533:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487419904, count: 4096 00000080:00200000:2.0:1788201707.118544:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487419904, bytes: 4096 00000080:00200000:2.0:1788201707.118547:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.118550:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.118552:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487419904, 487424000) 0 0 00000080:00200000:2.0:1788201707.118554:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.118559:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [118999, 118999] 00020000:00200000:2.0:1788201707.118562:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487419904 chunk: [487419904, 487424000) 487424000, 4096 00020000:00200000:2.0:1788201707.118563:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.118574:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [118999, 118999] 00000008:00010000:2.0:1788201707.118579:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.118581:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.118588:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118596:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487419904 487423999) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118605:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118612:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118619:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.118624:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487419904, 487424000) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.118628:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 118999 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.118632:0:17463:0:(rw26.c:898:ll_write_end()) pos 487419904, len 4096, copied 4096 00000080:00200000:2.0:1788201707.118634:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000005e3f1f3[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 118999] 00000080:00200000:2.0:1788201707.118637:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000aaa2a1fa fffffc0001029 2:0 ffff93c0c4403b60 118999 lru 00000080:00200000:2.0:1788201707.118641:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000056e093e2 118999: 1< 2 - - > 2< 487419904 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70584 - -> 00000080:00200000:2.0:1788201707.118642:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000005e3f1f3 00000080:00200000:2.0:1788201707.118643:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.118644:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.118650:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.118658:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.118660:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.118661:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.118668:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.118669:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.118671:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.118673:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118678:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.118683:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.118684:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.118687:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.118689:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [118999, 118999] 00000080:00200000:2.0:1788201707.118696:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.118698:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.118716:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.118718:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.118728:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487424000, count: 4096 00000080:00200000:2.0:1788201707.118734:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487424000, bytes: 4096 00000080:00200000:2.0:1788201707.118736:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.118738:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.118739:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487424000, 487428096) 0 0 00000080:00200000:2.0:1788201707.118740:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.118744:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119000, 119000] 00020000:00200000:2.0:1788201707.118746:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487424000 chunk: [487424000, 487428096) 487428096, 4096 00020000:00200000:2.0:1788201707.118747:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.118753:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119000, 119000] 00000008:00010000:2.0:1788201707.118756:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.118758:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.118763:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118768:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487424000 487428095) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118774:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118778:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118782:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.118786:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487424000, 487428096) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.118789:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119000 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.118793:0:17463:0:(rw26.c:898:ll_write_end()) pos 487424000, len 4096, copied 4096 00000080:00200000:2.0:1788201707.118795:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006193d599[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 119000] 00000080:00200000:2.0:1788201707.118797:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c260030e fffffc0001029 2:0 ffff93c0c4403c40 119000 lru 00000080:00200000:2.0:1788201707.118801:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000f2e18e84 119000: 1< 2 - - > 2< 487424000 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70585 - -> 00000080:00200000:2.0:1788201707.118802:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006193d599 00000080:00200000:2.0:1788201707.118803:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.118804:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.118810:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.118817:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.118819:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.118820:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.118826:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.118827:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.118847:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.118850:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118855:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.118861:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.118862:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.118865:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.118868:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119000, 119000] 00000080:00200000:2.0:1788201707.118876:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.118878:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.118897:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.118899:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.118909:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487428096, count: 4096 00000080:00200000:2.0:1788201707.118915:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487428096, bytes: 4096 00000080:00200000:2.0:1788201707.118917:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.118919:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.118920:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487428096, 487432192) 0 0 00000080:00200000:2.0:1788201707.118921:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.118925:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119001, 119001] 00020000:00200000:2.0:1788201707.118927:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487428096 chunk: [487428096, 487432192) 487432192, 4096 00020000:00200000:2.0:1788201707.118928:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.118935:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119001, 119001] 00000008:00010000:2.0:1788201707.118938:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.118939:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.118944:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118950:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487428096 487432191) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118955:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118959:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.118963:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.118967:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487428096, 487432192) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.118971:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119001 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.118974:0:17463:0:(rw26.c:898:ll_write_end()) pos 487428096, len 4096, copied 4096 00000080:00200000:2.0:1788201707.118977:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002834c7bc[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 119001] 00000080:00200000:2.0:1788201707.118978:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d98c7e6f fffffc0001029 2:0 ffff93c0c4403540 119001 lru 00000080:00200000:2.0:1788201707.118982:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000000035677c 119001: 1< 2 - - > 2< 487428096 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70586 - -> 00000080:00200000:2.0:1788201707.118984:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002834c7bc 00000080:00200000:2.0:1788201707.118984:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.118985:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.118991:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.118999:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.119000:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.119002:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.119021:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.119023:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.119025:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.119028:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119033:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.119038:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.119039:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.119042:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.119044:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119001, 119001] 00000080:00200000:2.0:1788201707.119051:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.119053:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.119072:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.119074:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.119084:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487432192, count: 4096 00000080:00200000:2.0:1788201707.119090:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487432192, bytes: 4096 00000080:00200000:2.0:1788201707.119092:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.119094:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.119095:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487432192, 487436288) 0 0 00000080:00200000:2.0:1788201707.119097:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.119100:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119002, 119002] 00020000:00200000:2.0:1788201707.119102:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487432192 chunk: [487432192, 487436288) 487436288, 4096 00020000:00200000:2.0:1788201707.119103:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.119109:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119002, 119002] 00000008:00010000:2.0:1788201707.119113:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.119114:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.119119:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119125:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487432192 487436287) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119130:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119134:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119138:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.119146:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487432192, 487436288) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.119149:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119002 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.119152:0:17463:0:(rw26.c:898:ll_write_end()) pos 487432192, len 4096, copied 4096 00000080:00200000:2.0:1788201707.119155:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000066f5eccb[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 119002] 00000080:00200000:2.0:1788201707.119156:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f65701c9 fffffc0001029 2:0 ffff93c0c4403460 119002 lru 00000080:00200000:2.0:1788201707.119160:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e0893bb8 119002: 1< 2 - - > 2< 487432192 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70587 - -> 00000080:00200000:2.0:1788201707.119161:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000066f5eccb 00000080:00200000:2.0:1788201707.119162:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.119163:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.119169:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.119176:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.119178:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.119179:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.119185:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.119187:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.119188:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.119191:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119195:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.119200:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.119201:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.119204:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.119206:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119002, 119002] 00000080:00200000:2.0:1788201707.119213:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.119217:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.119243:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.119247:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.119263:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487436288, count: 4096 00000080:00200000:2.0:1788201707.119273:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487436288, bytes: 4096 00000080:00200000:2.0:1788201707.119274:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.119277:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.119278:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487436288, 487440384) 0 0 00000080:00200000:2.0:1788201707.119279:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.119283:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119003, 119003] 00020000:00200000:2.0:1788201707.119285:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487436288 chunk: [487436288, 487440384) 487440384, 4096 00020000:00200000:2.0:1788201707.119286:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.119293:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119003, 119003] 00000008:00010000:2.0:1788201707.119296:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.119297:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.119303:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119308:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487436288 487440383) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119314:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119318:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119326:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.119330:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487436288, 487440384) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.119333:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119003 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.119337:0:17463:0:(rw26.c:898:ll_write_end()) pos 487436288, len 4096, copied 4096 00000080:00200000:2.0:1788201707.119339:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009bab5785[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 119003] 00000080:00200000:2.0:1788201707.119341:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005827a153 fffffc0001029 2:0 ffff93c0c44042a0 119003 lru 00000080:00200000:2.0:1788201707.119345:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ea5dcdec 119003: 1< 2 - - > 2< 487436288 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70588 - -> 00000080:00200000:2.0:1788201707.119346:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009bab5785 00000080:00200000:2.0:1788201707.119347:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.119348:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.119354:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.119361:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.119364:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.119365:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.119371:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.119373:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.119374:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.119377:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119381:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.119390:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.119391:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.119394:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.119396:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119003, 119003] 00000080:00200000:2.0:1788201707.119402:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.119404:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.119422:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.119424:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.119434:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487440384, count: 4096 00000080:00200000:2.0:1788201707.119440:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487440384, bytes: 4096 00000080:00200000:2.0:1788201707.119442:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.119444:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.119445:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487440384, 487444480) 0 0 00000080:00200000:2.0:1788201707.119446:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.119449:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119004, 119004] 00020000:00200000:2.0:1788201707.119451:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487440384 chunk: [487440384, 487444480) 487444480, 4096 00020000:00200000:2.0:1788201707.119453:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.119459:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119004, 119004] 00000008:00010000:2.0:1788201707.119462:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.119464:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.119469:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119475:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487440384 487444479) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119480:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119489:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119493:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.119497:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487440384, 487444480) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.119499:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119004 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.119503:0:17463:0:(rw26.c:898:ll_write_end()) pos 487440384, len 4096, copied 4096 00000080:00200000:2.0:1788201707.119505:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ff4f6971[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 119004] 00000080:00200000:2.0:1788201707.119507:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f09632ea fffffc0001029 2:0 ffff93c0c4404e00 119004 lru 00000080:00200000:2.0:1788201707.119510:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000622cd3e5 119004: 1< 2 - - > 2< 487440384 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70589 - -> 00000080:00200000:2.0:1788201707.119511:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ff4f6971 00000080:00200000:2.0:1788201707.119512:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.119513:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.119520:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.119532:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.119534:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.119536:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.119546:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.119548:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.119551:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.119554:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119566:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.119573:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.119575:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.119579:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.119582:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119004, 119004] 00000080:00200000:2.0:1788201707.119592:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.119595:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.119620:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.119623:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.119637:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487444480, count: 4096 00000080:00200000:2.0:1788201707.119645:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487444480, bytes: 4096 00000080:00200000:2.0:1788201707.119648:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.119651:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.119652:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487444480, 487448576) 0 0 00000080:00200000:2.0:1788201707.119655:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.119659:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119005, 119005] 00020000:00200000:2.0:1788201707.119661:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487444480 chunk: [487444480, 487448576) 487448576, 4096 00020000:00200000:2.0:1788201707.119664:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.119673:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119005, 119005] 00000008:00010000:2.0:1788201707.119677:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.119679:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.119686:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119694:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487444480 487448575) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119708:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119715:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119719:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.119724:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487444480, 487448576) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.119728:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119005 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.119732:0:17463:0:(rw26.c:898:ll_write_end()) pos 487444480, len 4096, copied 4096 00000080:00200000:2.0:1788201707.119734:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002802249e[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 119005] 00000080:00200000:2.0:1788201707.119736:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c218f846 fffffc0001029 2:0 ffff93c0c44041c0 119005 lru 00000080:00200000:2.0:1788201707.119740:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007d3f4a07 119005: 1< 2 - - > 2< 487444480 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70590 - -> 00000080:00200000:2.0:1788201707.119741:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002802249e 00000080:00200000:2.0:1788201707.119742:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.119743:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.119749:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.119757:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.119759:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.119760:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.119767:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.119768:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.119770:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.119773:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119777:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.119782:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.119784:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.119786:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.119789:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119005, 119005] 00000080:00200000:2.0:1788201707.119795:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.119797:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.119815:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.119818:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.119828:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487448576, count: 4096 00000080:00200000:2.0:1788201707.119847:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487448576, bytes: 4096 00000080:00200000:2.0:1788201707.119849:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.119851:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.119852:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487448576, 487452672) 0 0 00000080:00200000:2.0:1788201707.119854:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.119857:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119006, 119006] 00020000:00200000:2.0:1788201707.119859:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487448576 chunk: [487448576, 487452672) 487452672, 4096 00020000:00200000:2.0:1788201707.119860:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.119867:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119006, 119006] 00000008:00010000:2.0:1788201707.119870:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.119871:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.119876:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119885:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487448576 487452671) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119891:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119895:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119899:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.119903:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487448576, 487452672) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.119907:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119006 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.119910:0:17463:0:(rw26.c:898:ll_write_end()) pos 487448576, len 4096, copied 4096 00000080:00200000:2.0:1788201707.119912:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c3f5294a[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 119006] 00000080:00200000:2.0:1788201707.119914:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f5a15ed1 fffffc0001029 2:0 ffff93c0c4404620 119006 lru 00000080:00200000:2.0:1788201707.119918:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d8ed3426 119006: 1< 2 - - > 2< 487448576 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70591 - -> 00000080:00200000:2.0:1788201707.119919:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c3f5294a 00000080:00200000:2.0:1788201707.119920:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.119921:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.119926:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.119934:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.119935:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.119937:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.119943:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.119945:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.119946:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.119948:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.119953:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.119958:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.119960:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.119962:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.119964:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119006, 119006] 00000080:00200000:2.0:1788201707.119970:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.119972:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.119990:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.119992:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.120015:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487452672, count: 4096 00000080:00200000:2.0:1788201707.120028:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487452672, bytes: 4096 00000080:00200000:2.0:1788201707.120030:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.120034:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.120036:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487452672, 487456768) 0 0 00000080:00200000:2.0:1788201707.120038:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.120043:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119007, 119007] 00020000:00200000:2.0:1788201707.120046:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487452672 chunk: [487452672, 487456768) 487456768, 4096 00020000:00200000:2.0:1788201707.120049:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.120059:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119007, 119007] 00000008:00010000:2.0:1788201707.120063:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.120064:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.120070:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120076:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487452672 487456767) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120082:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120086:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120090:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.120095:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487452672, 487456768) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.120098:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119007 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.120102:0:17463:0:(rw26.c:898:ll_write_end()) pos 487452672, len 4096, copied 4096 00000080:00200000:2.0:1788201707.120105:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ffcda072[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 119007] 00000080:00200000:2.0:1788201707.120107:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000006fe0164e fffffc0001029 2:0 ffff93c0c4404d20 119007 lru 00000080:00200000:2.0:1788201707.120110:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000002e57ac58 119007: 1< 2 - - > 2< 487452672 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70592 - -> 00000080:00200000:2.0:1788201707.120112:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ffcda072 00000080:00200000:2.0:1788201707.120112:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.120114:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.120119:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.120128:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.120130:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.120131:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.120137:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.120139:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.120141:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.120143:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120148:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.120153:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.120154:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.120157:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.120159:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119007, 119007] 00000080:00200000:2.0:1788201707.120166:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.120167:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.120185:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.120188:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.120198:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487456768, count: 4096 00000080:00200000:2.0:1788201707.120204:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487456768, bytes: 4096 00000080:00200000:2.0:1788201707.120205:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.120207:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.120209:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487456768, 487460864) 0 0 00000080:00200000:2.0:1788201707.120210:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.120215:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119008, 119008] 00020000:00200000:2.0:1788201707.120217:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487456768 chunk: [487456768, 487460864) 487460864, 4096 00020000:00200000:2.0:1788201707.120220:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.120231:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119008, 119008] 00000008:00010000:2.0:1788201707.120236:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.120238:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.120245:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120255:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487456768 487460863) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120264:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120270:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120274:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.120279:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487456768, 487460864) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.120282:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119008 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.120286:0:17463:0:(rw26.c:898:ll_write_end()) pos 487456768, len 4096, copied 4096 00000080:00200000:2.0:1788201707.120288:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007dc79e3c[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 119008] 00000080:00200000:2.0:1788201707.120290:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c29e44a1 fffffc0001029 2:0 ffff93c0c44049a0 119008 lru 00000080:00200000:2.0:1788201707.120293:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d417a03c 119008: 1< 2 - - > 2< 487456768 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70593 - -> 00000080:00200000:2.0:1788201707.120294:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007dc79e3c 00000080:00200000:2.0:1788201707.120295:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.120297:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.120302:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.120314:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.120315:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.120317:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.120323:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.120324:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.120326:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.120329:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120334:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.120339:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.120340:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.120343:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.120345:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119008, 119008] 00000080:00200000:2.0:1788201707.120351:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.120353:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.120371:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.120374:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.120384:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487460864, count: 4096 00000080:00200000:2.0:1788201707.120390:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487460864, bytes: 4096 00000080:00200000:2.0:1788201707.120392:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.120394:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.120395:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487460864, 487464960) 0 0 00000080:00200000:2.0:1788201707.120397:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.120400:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119009, 119009] 00020000:00200000:2.0:1788201707.120402:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487460864 chunk: [487460864, 487464960) 487464960, 4096 00020000:00200000:2.0:1788201707.120404:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.120410:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119009, 119009] 00000008:00010000:2.0:1788201707.120413:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.120414:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.120420:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120425:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487460864 487464959) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120430:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120435:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120439:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.120443:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487460864, 487464960) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.120446:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119009 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.120449:0:17463:0:(rw26.c:898:ll_write_end()) pos 487460864, len 4096, copied 4096 00000080:00200000:2.0:1788201707.120451:0:17463:0:(rw26.c:956:ll_write_end()) page@000000002013d137[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 119009] 00000080:00200000:2.0:1788201707.120453:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000050277918 fffffc0001029 2:0 ffff93c0c4404b60 119009 lru 00000080:00200000:2.0:1788201707.120457:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000008dd1d3d8 119009: 1< 2 - - > 2< 487460864 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70594 - -> 00000080:00200000:2.0:1788201707.120459:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000002013d137 00000080:00200000:2.0:1788201707.120459:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.120461:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.120466:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.120474:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.120475:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.120476:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.120482:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.120484:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.120486:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.120488:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120492:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.120497:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.120498:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.120501:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.120502:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119009, 119009] 00000080:00200000:2.0:1788201707.120509:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.120511:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.120537:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.120540:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.120555:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487464960, count: 4096 00000080:00200000:2.0:1788201707.120565:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487464960, bytes: 4096 00000080:00200000:2.0:1788201707.120568:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.120571:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.120572:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487464960, 487469056) 0 0 00000080:00200000:2.0:1788201707.120575:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.120580:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119010, 119010] 00020000:00200000:2.0:1788201707.120583:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487464960 chunk: [487464960, 487469056) 487469056, 4096 00020000:00200000:2.0:1788201707.120585:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.120593:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119010, 119010] 00000008:00010000:2.0:1788201707.120597:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.120598:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.120603:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120609:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487464960 487469055) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120615:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120619:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120623:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.120628:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487464960, 487469056) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.120632:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119010 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.120635:0:17463:0:(rw26.c:898:ll_write_end()) pos 487464960, len 4096, copied 4096 00000080:00200000:2.0:1788201707.120638:0:17463:0:(rw26.c:956:ll_write_end()) page@000000007db3e4fa[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 119010] 00000080:00200000:2.0:1788201707.120640:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cd576cc1 fffffc0001029 2:0 ffff93c0c4404c40 119010 lru 00000080:00200000:2.0:1788201707.120643:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000066f99aa7 119010: 1< 2 - - > 2< 487464960 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70595 - -> 00000080:00200000:2.0:1788201707.120646:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000007db3e4fa 00000080:00200000:2.0:1788201707.120647:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.120648:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.120654:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.120661:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.120663:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.120664:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.120671:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.120672:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.120674:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.120676:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120681:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.120686:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.120687:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.120689:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.120691:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119010, 119010] 00000080:00200000:2.0:1788201707.120698:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.120700:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1788201707.120718:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:2.0:1788201707.120720:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1788201707.120730:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487469056, count: 4096 00000080:00200000:2.0:1788201707.120736:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487469056, bytes: 4096 00000080:00200000:2.0:1788201707.120738:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1788201707.120740:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:2.0:1788201707.120742:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487469056, 487473152) 0 0 00000080:00200000:2.0:1788201707.120743:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788201707.120746:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119011, 119011] 00020000:00200000:2.0:1788201707.120748:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487469056 chunk: [487469056, 487473152) 487473152, 4096 00020000:00200000:2.0:1788201707.120749:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201707.120756:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119011, 119011] 00000008:00010000:2.0:1788201707.120759:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788201707.120760:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d324a030, osc lock ffff93c1c82590a0, flags 0x0 00010000:00010000:2.0:1788201707.120765:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120771:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487469056 487473151) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120776:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120781:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120784:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.120788:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487469056, 487473152) append 0 append lockpos 0 00000080:00200000:2.0:1788201707.120791:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119011 of 0 to 4096 bytes 00000080:00200000:2.0:1788201707.120795:0:17463:0:(rw26.c:898:ll_write_end()) pos 487469056, len 4096, copied 4096 00000080:00200000:2.0:1788201707.120797:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000028851203[3 0000000075f4bf38 2 1 0000000050abd81d 0 0 119011] 00000080:00200000:2.0:1788201707.120799:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a58a00ab fffffc0001029 2:0 ffff93c0c4404540 119011 lru 00000080:00200000:2.0:1788201707.120803:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000d8563f73 119011: 1< 2 - - > 2< 487469056 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70596 - -> 00000080:00200000:2.0:1788201707.120804:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000028851203 00000080:00200000:2.0:1788201707.120805:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1788201707.120806:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1788201707.120811:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1788201707.120818:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:2.0:1788201707.120820:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1788201707.120821:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1788201707.120827:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_release():245 00000020:00010000:2.0:1788201707.120845:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788201707.120847:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d324a030 (ffff93c1d300a878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788201707.120850:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1788201707.120855:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1788201707.120861:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c5702d58 (ffff93c1d300a368/0) at cl_lock_fini():72 00000020:00010000:2.0:1788201707.120862:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d324a030 (ffff93c1d300a368/1) at cl_lock_fini():72 00000080:00200000:2.0:1788201707.120865:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1788201707.120868:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119011, 119011] 00000080:00200000:2.0:1788201707.120877:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201707.120879:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.120938:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.120943:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.120962:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487473152, count: 4096 00000080:00200000:3.0:1788201707.120977:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487473152, bytes: 4096 00000080:00200000:3.0:1788201707.120979:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.120982:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.120983:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487473152, 487477248) 0 0 00000080:00200000:3.0:1788201707.120986:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.120992:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119012, 119012] 00020000:00200000:3.0:1788201707.120995:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487473152 chunk: [487473152, 487477248) 487477248, 4096 00020000:00200000:3.0:1788201707.120998:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.121027:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119012, 119012] 00000008:00010000:3.0:1788201707.121035:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.121038:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.121047:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121056:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487473152 487477247) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121062:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121067:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121072:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.121077:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487473152, 487477248) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.121081:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119012 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.121085:0:17463:0:(rw26.c:898:ll_write_end()) pos 487473152, len 4096, copied 4096 00000080:00200000:3.0:1788201707.121089:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c995d25d[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119012] 00000080:00200000:3.0:1788201707.121091:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000064f77b6b fffffc0001029 2:0 ffff93c0c4404460 119012 lru 00000080:00200000:3.0:1788201707.121095:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000019bedddf 119012: 1< 2 - - > 2< 487473152 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70597 - -> 00000080:00200000:3.0:1788201707.121096:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c995d25d 00000080:00200000:3.0:1788201707.121097:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.121098:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.121105:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.121119:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.121121:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.121122:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.121131:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.121133:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.121135:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.121138:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121143:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.121149:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.121150:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.121153:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.121155:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119012, 119012] 00000080:00200000:3.0:1788201707.121171:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.121173:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.121204:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.121208:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.121230:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487477248, count: 4096 00000080:00200000:3.0:1788201707.121239:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487477248, bytes: 4096 00000080:00200000:3.0:1788201707.121241:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.121243:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.121245:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487477248, 487481344) 0 0 00000080:00200000:3.0:1788201707.121246:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.121250:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119013, 119013] 00020000:00200000:3.0:1788201707.121253:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487477248 chunk: [487477248, 487481344) 487481344, 4096 00020000:00200000:3.0:1788201707.121254:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.121262:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119013, 119013] 00000008:00010000:3.0:1788201707.121266:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.121268:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.121274:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121280:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487477248 487481343) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121286:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121291:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121298:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.121305:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487477248, 487481344) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.121310:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119013 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.121315:0:17463:0:(rw26.c:898:ll_write_end()) pos 487477248, len 4096, copied 4096 00000080:00200000:3.0:1788201707.121320:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005e31db5e[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119013] 00000080:00200000:3.0:1788201707.121323:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f3ae7140 fffffc0001029 2:0 ffff93c0c44040e0 119013 lru 00000080:00200000:3.0:1788201707.121330:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000954a5218 119013: 1< 2 - - > 2< 487477248 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70598 - -> 00000080:00200000:3.0:1788201707.121332:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005e31db5e 00000080:00200000:3.0:1788201707.121333:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.121336:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.121344:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.121354:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.121356:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.121357:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.121364:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.121366:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.121368:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.121370:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121376:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.121381:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.121382:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.121384:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.121386:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119013, 119013] 00000080:00200000:3.0:1788201707.121395:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.121397:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.121423:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.121426:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.121438:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487481344, count: 4096 00000080:00200000:3.0:1788201707.121444:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487481344, bytes: 4096 00000080:00200000:3.0:1788201707.121446:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.121449:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.121450:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487481344, 487485440) 0 0 00000080:00200000:3.0:1788201707.121452:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.121455:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119014, 119014] 00020000:00200000:3.0:1788201707.121458:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487481344 chunk: [487481344, 487485440) 487485440, 4096 00020000:00200000:3.0:1788201707.121459:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.121466:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119014, 119014] 00000008:00010000:3.0:1788201707.121470:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.121472:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.121477:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121483:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487481344 487485439) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121490:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121497:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121504:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.121512:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487481344, 487485440) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.121517:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119014 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.121522:0:17463:0:(rw26.c:898:ll_write_end()) pos 487481344, len 4096, copied 4096 00000080:00200000:3.0:1788201707.121526:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000088e90caa[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119014] 00000080:00200000:3.0:1788201707.121529:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000cc0e0b99 fffffc0001029 2:0 ffff93c0c4404700 119014 lru 00000080:00200000:3.0:1788201707.121536:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000753fbf04 119014: 1< 2 - - > 2< 487481344 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70599 - -> 00000080:00200000:3.0:1788201707.121539:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000088e90caa 00000080:00200000:3.0:1788201707.121541:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.121543:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.121550:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.121559:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.121562:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.121563:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.121570:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.121572:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.121574:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.121576:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121581:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.121586:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.121588:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.121590:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.121592:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119014, 119014] 00000080:00200000:3.0:1788201707.121604:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.121607:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.121645:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.121648:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.121660:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487485440, count: 4096 00000080:00200000:3.0:1788201707.121667:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487485440, bytes: 4096 00000080:00200000:3.0:1788201707.121669:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.121672:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.121673:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487485440, 487489536) 0 0 00000080:00200000:3.0:1788201707.121675:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.121678:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119015, 119015] 00020000:00200000:3.0:1788201707.121680:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487485440 chunk: [487485440, 487489536) 487489536, 4096 00020000:00200000:3.0:1788201707.121682:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.121689:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119015, 119015] 00000008:00010000:3.0:1788201707.121692:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.121694:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.121700:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121706:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487485440 487489535) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121711:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121716:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121720:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.121725:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487485440, 487489536) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.121728:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119015 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.121732:0:17463:0:(rw26.c:898:ll_write_end()) pos 487485440, len 4096, copied 4096 00000080:00200000:3.0:1788201707.121734:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000ffe00ad3[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119015] 00000080:00200000:3.0:1788201707.121737:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000045c0f329 fffffc0001029 2:0 ffff93c0c4404000 119015 lru 00000080:00200000:3.0:1788201707.121741:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000016ec9da2 119015: 1< 2 - - > 2< 487485440 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70600 - -> 00000080:00200000:3.0:1788201707.121742:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000ffe00ad3 00000080:00200000:3.0:1788201707.121750:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.121752:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.121758:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.121767:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.121768:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.121770:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.121777:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.121778:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.121780:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.121782:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121789:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.121795:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.121797:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.121799:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.121801:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119015, 119015] 00000080:00200000:3.0:1788201707.121809:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.121811:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.121843:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.121846:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.121858:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487489536, count: 4096 00000080:00200000:3.0:1788201707.121864:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487489536, bytes: 4096 00000080:00200000:3.0:1788201707.121866:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.121869:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.121870:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487489536, 487493632) 0 0 00000080:00200000:3.0:1788201707.121871:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.121875:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119016, 119016] 00020000:00200000:3.0:1788201707.121876:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487489536 chunk: [487489536, 487493632) 487493632, 4096 00020000:00200000:3.0:1788201707.121878:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.121885:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119016, 119016] 00000008:00010000:3.0:1788201707.121888:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.121890:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.121895:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121901:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487489536 487493631) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121906:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121911:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121915:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.121919:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487489536, 487493632) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.121922:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119016 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.121927:0:17463:0:(rw26.c:898:ll_write_end()) pos 487489536, len 4096, copied 4096 00000080:00200000:3.0:1788201707.121929:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000064cb4153[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119016] 00000080:00200000:3.0:1788201707.121931:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000d8f1d10 fffffc0001029 2:0 ffff93c0c4404a80 119016 lru 00000080:00200000:3.0:1788201707.121935:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000016ebb495 119016: 1< 2 - - > 2< 487489536 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70601 - -> 00000080:00200000:3.0:1788201707.121936:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000064cb4153 00000080:00200000:3.0:1788201707.121938:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.121939:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.121944:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.121952:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.121953:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.121955:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.121961:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.121963:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.121964:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.121967:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.121972:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.121977:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.121978:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.121980:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.121982:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119016, 119016] 00000080:00200000:3.0:1788201707.121991:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.121994:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.122037:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.122042:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.122056:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487493632, count: 4096 00000080:00200000:3.0:1788201707.122063:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487493632, bytes: 4096 00000080:00200000:3.0:1788201707.122065:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.122068:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.122069:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487493632, 487497728) 0 0 00000080:00200000:3.0:1788201707.122070:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.122074:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119017, 119017] 00020000:00200000:3.0:1788201707.122076:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487493632 chunk: [487493632, 487497728) 487497728, 4096 00020000:00200000:3.0:1788201707.122077:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.122085:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119017, 119017] 00000008:00010000:3.0:1788201707.122088:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.122089:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.122095:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122101:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487493632 487497727) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122106:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122111:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122115:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.122119:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487493632, 487497728) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.122123:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119017 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.122127:0:17463:0:(rw26.c:898:ll_write_end()) pos 487493632, len 4096, copied 4096 00000080:00200000:3.0:1788201707.122129:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005ec48ea7[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119017] 00000080:00200000:3.0:1788201707.122131:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000007f1c517c fffffc0001029 2:0 ffff93c0c4404380 119017 lru 00000080:00200000:3.0:1788201707.122135:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000049aff52a 119017: 1< 2 - - > 2< 487493632 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70602 - -> 00000080:00200000:3.0:1788201707.122136:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005ec48ea7 00000080:00200000:3.0:1788201707.122137:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.122138:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.122144:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.122152:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.122154:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.122155:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.122162:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.122163:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.122165:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.122167:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122171:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.122177:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.122179:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.122183:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.122186:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119017, 119017] 00000080:00200000:3.0:1788201707.122197:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.122201:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.122228:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.122231:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.122242:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487497728, count: 4096 00000080:00200000:3.0:1788201707.122248:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487497728, bytes: 4096 00000080:00200000:3.0:1788201707.122250:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.122252:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.122254:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487497728, 487501824) 0 0 00000080:00200000:3.0:1788201707.122255:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.122258:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119018, 119018] 00020000:00200000:3.0:1788201707.122261:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487497728 chunk: [487497728, 487501824) 487501824, 4096 00020000:00200000:3.0:1788201707.122262:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.122269:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119018, 119018] 00000008:00010000:3.0:1788201707.122272:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.122274:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.122279:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122285:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487497728 487501823) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122290:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122296:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122303:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.122315:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487497728, 487501824) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.122320:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119018 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.122325:0:17463:0:(rw26.c:898:ll_write_end()) pos 487497728, len 4096, copied 4096 00000080:00200000:3.0:1788201707.122330:0:17463:0:(rw26.c:956:ll_write_end()) page@000000003a9c946b[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119018] 00000080:00200000:3.0:1788201707.122333:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000009c748765 fffffc0001029 2:0 ffff93c0c44048c0 119018 lru 00000080:00200000:3.0:1788201707.122339:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000003555611d 119018: 1< 2 - - > 2< 487497728 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70603 - -> 00000080:00200000:3.0:1788201707.122341:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000003a9c946b 00000080:00200000:3.0:1788201707.122342:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.122343:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.122350:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.122358:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.122360:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.122362:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.122368:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.122370:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.122372:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.122374:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122379:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.122384:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.122385:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.122388:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.122390:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119018, 119018] 00000080:00200000:3.0:1788201707.122398:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.122400:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.122418:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.122420:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.122430:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487501824, count: 4096 00000080:00200000:3.0:1788201707.122436:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487501824, bytes: 4096 00000080:00200000:3.0:1788201707.122438:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.122440:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.122441:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487501824, 487505920) 0 0 00000080:00200000:3.0:1788201707.122443:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.122446:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119019, 119019] 00020000:00200000:3.0:1788201707.122448:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487501824 chunk: [487501824, 487505920) 487505920, 4096 00020000:00200000:3.0:1788201707.122449:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.122456:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119019, 119019] 00000008:00010000:3.0:1788201707.122459:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.122461:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.122466:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122472:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487501824 487505919) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122477:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122482:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122490:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.122497:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487501824, 487505920) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.122502:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119019 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.122506:0:17463:0:(rw26.c:898:ll_write_end()) pos 487501824, len 4096, copied 4096 00000080:00200000:3.0:1788201707.122510:0:17463:0:(rw26.c:956:ll_write_end()) page@000000005f70c1dc[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119019] 00000080:00200000:3.0:1788201707.122513:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bff28895 fffffc0001029 2:0 ffff93c0c4404ee0 119019 lru 00000080:00200000:3.0:1788201707.122520:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000019f703c1 119019: 1< 2 - - > 2< 487501824 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70604 - -> 00000080:00200000:3.0:1788201707.122522:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000005f70c1dc 00000080:00200000:3.0:1788201707.122523:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.122525:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.122534:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.122544:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.122546:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.122547:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.122554:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.122555:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.122557:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.122560:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122564:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.122573:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.122575:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.122577:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.122579:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119019, 119019] 00000080:00200000:3.0:1788201707.122587:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.122589:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.122614:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.122618:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.122635:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487505920, count: 4096 00000080:00200000:3.0:1788201707.122646:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487505920, bytes: 4096 00000080:00200000:3.0:1788201707.122648:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.122650:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.122651:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487505920, 487510016) 0 0 00000080:00200000:3.0:1788201707.122653:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.122656:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119020, 119020] 00020000:00200000:3.0:1788201707.122658:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487505920 chunk: [487505920, 487510016) 487510016, 4096 00020000:00200000:3.0:1788201707.122660:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.122667:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119020, 119020] 00000008:00010000:3.0:1788201707.122670:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.122672:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.122677:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122683:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487505920 487510015) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122689:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122697:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122701:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.122705:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487505920, 487510016) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.122708:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119020 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.122712:0:17463:0:(rw26.c:898:ll_write_end()) pos 487505920, len 4096, copied 4096 00000080:00200000:3.0:1788201707.122714:0:17463:0:(rw26.c:956:ll_write_end()) page@000000009529d803[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119020] 00000080:00200000:3.0:1788201707.122716:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000d93cb58c fffffc0001029 2:0 ffff93c0c44047e0 119020 lru 00000080:00200000:3.0:1788201707.122719:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000073959c7e 119020: 1< 2 - - > 2< 487505920 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70605 - -> 00000080:00200000:3.0:1788201707.122720:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000009529d803 00000080:00200000:3.0:1788201707.122721:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.122722:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.122728:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.122735:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.122737:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.122738:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.122745:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.122747:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.122748:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.122750:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122759:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.122763:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.122765:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.122767:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.122769:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119020, 119020] 00000080:00200000:3.0:1788201707.122776:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.122778:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.122802:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.122805:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.122820:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487510016, count: 4096 00000080:00200000:3.0:1788201707.122849:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487510016, bytes: 4096 00000080:00200000:3.0:1788201707.122852:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.122855:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.122856:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487510016, 487514112) 0 0 00000080:00200000:3.0:1788201707.122857:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.122861:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119021, 119021] 00020000:00200000:3.0:1788201707.122863:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487510016 chunk: [487510016, 487514112) 487514112, 4096 00020000:00200000:3.0:1788201707.122865:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.122872:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119021, 119021] 00000008:00010000:3.0:1788201707.122876:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.122877:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.122882:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122888:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487510016 487514111) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122897:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122902:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122906:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.122910:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487510016, 487514112) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.122914:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119021 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.122917:0:17463:0:(rw26.c:898:ll_write_end()) pos 487510016, len 4096, copied 4096 00000080:00200000:3.0:1788201707.122920:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000887e8c8d[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119021] 00000080:00200000:3.0:1788201707.122922:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000629f9ff4 fffffc0001029 2:0 ffff93c0c44059a0 119021 lru 00000080:00200000:3.0:1788201707.122926:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b5e1702f 119021: 1< 2 - - > 2< 487510016 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70606 - -> 00000080:00200000:3.0:1788201707.122927:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000887e8c8d 00000080:00200000:3.0:1788201707.122928:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.122929:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.122935:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.122943:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.122945:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.122946:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.122952:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.122954:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.122956:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.122958:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.122963:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.122968:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.122970:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.122972:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.122974:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119021, 119021] 00000080:00200000:3.0:1788201707.122981:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.122983:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.123019:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.123022:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.123040:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487514112, count: 4096 00000080:00200000:3.0:1788201707.123051:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487514112, bytes: 4096 00000080:00200000:3.0:1788201707.123053:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.123055:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.123056:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487514112, 487518208) 0 0 00000080:00200000:3.0:1788201707.123058:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.123061:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119022, 119022] 00020000:00200000:3.0:1788201707.123063:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487514112 chunk: [487514112, 487518208) 487518208, 4096 00020000:00200000:3.0:1788201707.123065:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.123072:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119022, 119022] 00000008:00010000:3.0:1788201707.123076:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.123077:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.123082:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123091:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487514112 487518207) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123097:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123102:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123106:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.123110:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487514112, 487518208) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.123113:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119022 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.123116:0:17463:0:(rw26.c:898:ll_write_end()) pos 487514112, len 4096, copied 4096 00000080:00200000:3.0:1788201707.123119:0:17463:0:(rw26.c:956:ll_write_end()) page@000000000dbf7c18[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119022] 00000080:00200000:3.0:1788201707.123121:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000e45eb382 fffffc0001029 2:0 ffff93c0c13bc460 119022 lru 00000080:00200000:3.0:1788201707.123124:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004a2a5718 119022: 1< 2 - - > 2< 487514112 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70607 - -> 00000080:00200000:3.0:1788201707.123125:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000000dbf7c18 00000080:00200000:3.0:1788201707.123126:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.123127:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.123133:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.123141:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.123142:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.123144:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.123150:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.123152:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.123153:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.123156:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123160:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.123166:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.123167:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.123169:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.123171:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119022, 119022] 00000080:00200000:3.0:1788201707.123181:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.123184:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.123212:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.123216:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.123232:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487518208, count: 4096 00000080:00200000:3.0:1788201707.123239:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487518208, bytes: 4096 00000080:00200000:3.0:1788201707.123241:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.123244:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.123245:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487518208, 487522304) 0 0 00000080:00200000:3.0:1788201707.123246:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.123250:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119023, 119023] 00020000:00200000:3.0:1788201707.123252:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487518208 chunk: [487518208, 487522304) 487522304, 4096 00020000:00200000:3.0:1788201707.123254:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.123261:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119023, 119023] 00000008:00010000:3.0:1788201707.123264:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.123265:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.123271:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123277:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487518208 487522303) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123282:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123286:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123290:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.123297:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487518208, 487522304) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.123301:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119023 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.123306:0:17463:0:(rw26.c:898:ll_write_end()) pos 487518208, len 4096, copied 4096 00000080:00200000:3.0:1788201707.123310:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000cfbc0010[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119023] 00000080:00200000:3.0:1788201707.123313:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000096a4c343 fffffc0001029 2:0 ffff93c0c13bc0e0 119023 lru 00000080:00200000:3.0:1788201707.123318:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000074dac66b 119023: 1< 2 - - > 2< 487518208 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70608 - -> 00000080:00200000:3.0:1788201707.123320:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000cfbc0010 00000080:00200000:3.0:1788201707.123322:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.123323:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.123332:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.123344:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.123346:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.123347:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.123354:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.123356:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.123358:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.123360:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123365:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.123371:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.123372:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.123374:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.123376:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119023, 119023] 00000080:00200000:3.0:1788201707.123384:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.123386:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.123404:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.123406:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.123417:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487522304, count: 4096 00000080:00200000:3.0:1788201707.123423:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487522304, bytes: 4096 00000080:00200000:3.0:1788201707.123424:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.123427:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.123428:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487522304, 487526400) 0 0 00000080:00200000:3.0:1788201707.123429:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.123433:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119024, 119024] 00020000:00200000:3.0:1788201707.123434:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487522304 chunk: [487522304, 487526400) 487526400, 4096 00020000:00200000:3.0:1788201707.123436:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.123443:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119024, 119024] 00000008:00010000:3.0:1788201707.123446:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.123448:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.123453:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123459:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487522304 487526399) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123464:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123468:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123472:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.123477:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487522304, 487526400) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.123480:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119024 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.123484:0:17463:0:(rw26.c:898:ll_write_end()) pos 487522304, len 4096, copied 4096 00000080:00200000:3.0:1788201707.123486:0:17463:0:(rw26.c:956:ll_write_end()) page@000000006aea64d1[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119024] 00000080:00200000:3.0:1788201707.123488:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000000ed55e7b fffffc0001029 2:0 ffff93c0c13bc700 119024 lru 00000080:00200000:3.0:1788201707.123494:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b3d152a4 119024: 1< 2 - - > 2< 487522304 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70609 - -> 00000080:00200000:3.0:1788201707.123496:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000006aea64d1 00000080:00200000:3.0:1788201707.123497:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.123499:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.123507:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.123523:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.123527:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.123529:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.123538:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.123540:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.123542:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.123545:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123550:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.123555:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.123556:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.123559:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.123561:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119024, 119024] 00000080:00200000:3.0:1788201707.123569:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.123571:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.123589:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.123591:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.123606:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487526400, count: 4096 00000080:00200000:3.0:1788201707.123616:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487526400, bytes: 4096 00000080:00200000:3.0:1788201707.123618:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.123622:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.123623:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487526400, 487530496) 0 0 00000080:00200000:3.0:1788201707.123626:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.123631:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119025, 119025] 00020000:00200000:3.0:1788201707.123634:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487526400 chunk: [487526400, 487530496) 487530496, 4096 00020000:00200000:3.0:1788201707.123636:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.123647:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119025, 119025] 00000008:00010000:3.0:1788201707.123650:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.123652:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.123657:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123663:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487526400 487530495) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123669:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123673:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123677:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.123682:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487526400, 487530496) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.123685:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119025 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.123688:0:17463:0:(rw26.c:898:ll_write_end()) pos 487526400, len 4096, copied 4096 00000080:00200000:3.0:1788201707.123691:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000f75ea446[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119025] 00000080:00200000:3.0:1788201707.123693:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f29ef396 fffffc0001029 2:0 ffff93c0c13bc000 119025 lru 00000080:00200000:3.0:1788201707.123697:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000b4198a1b 119025: 1< 2 - - > 2< 487526400 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70610 - -> 00000080:00200000:3.0:1788201707.123698:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000f75ea446 00000080:00200000:3.0:1788201707.123699:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.123700:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.123706:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.123713:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.123715:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.123716:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.123723:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.123724:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.123726:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.123728:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123733:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.123738:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.123739:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.123741:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.123743:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119025, 119025] 00000080:00200000:3.0:1788201707.123751:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.123752:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.123770:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.123772:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.123783:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487530496, count: 4096 00000080:00200000:3.0:1788201707.123792:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487530496, bytes: 4096 00000080:00200000:3.0:1788201707.123795:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.123798:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.123800:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487530496, 487534592) 0 0 00000080:00200000:3.0:1788201707.123802:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.123808:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119026, 119026] 00020000:00200000:3.0:1788201707.123810:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487530496 chunk: [487530496, 487534592) 487534592, 4096 00020000:00200000:3.0:1788201707.123813:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.123824:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119026, 119026] 00000008:00010000:3.0:1788201707.123842:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.123844:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.123850:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123856:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487530496 487534591) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123861:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123866:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123871:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.123875:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487530496, 487534592) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.123878:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119026 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.123882:0:17463:0:(rw26.c:898:ll_write_end()) pos 487530496, len 4096, copied 4096 00000080:00200000:3.0:1788201707.123884:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e863752b[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119026] 00000080:00200000:3.0:1788201707.123886:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c10ff004 fffffc0001029 2:0 ffff93c0c13bca80 119026 lru 00000080:00200000:3.0:1788201707.123889:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000007e7e34a8 119026: 1< 2 - - > 2< 487530496 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70611 - -> 00000080:00200000:3.0:1788201707.123892:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e863752b 00000080:00200000:3.0:1788201707.123893:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.123894:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.123899:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.123907:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.123909:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.123910:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.123916:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.123918:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.123920:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.123922:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.123927:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.123932:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.123933:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.123935:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.123937:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119026, 119026] 00000080:00200000:3.0:1788201707.123945:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.123947:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.123965:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.123967:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.123977:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487534592, count: 4096 00000080:00200000:3.0:1788201707.123983:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487534592, bytes: 4096 00000080:00200000:3.0:1788201707.123985:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.123988:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.123989:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487534592, 487538688) 0 0 00000080:00200000:3.0:1788201707.123992:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.123996:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119027, 119027] 00020000:00200000:3.0:1788201707.123999:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487534592 chunk: [487534592, 487538688) 487538688, 4096 00020000:00200000:3.0:1788201707.124001:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.124026:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119027, 119027] 00000008:00010000:3.0:1788201707.124031:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.124033:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.124041:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124049:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487534592 487538687) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124055:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124060:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124064:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.124069:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487534592, 487538688) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.124072:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119027 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.124076:0:17463:0:(rw26.c:898:ll_write_end()) pos 487534592, len 4096, copied 4096 00000080:00200000:3.0:1788201707.124078:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000236c1a73[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119027] 00000080:00200000:3.0:1788201707.124081:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000002b652d71 fffffc0001029 2:0 ffff93c0c13bc380 119027 lru 00000080:00200000:3.0:1788201707.124085:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001c7b610e 119027: 1< 2 - - > 2< 487534592 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70612 - -> 00000080:00200000:3.0:1788201707.124086:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000236c1a73 00000080:00200000:3.0:1788201707.124087:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.124088:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.124094:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.124101:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.124103:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.124104:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.124111:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.124113:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.124114:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.124117:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124122:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.124127:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.124128:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.124130:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.124132:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119027, 119027] 00000080:00200000:3.0:1788201707.124142:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.124146:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.124176:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.124179:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.124196:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487538688, count: 4096 00000080:00200000:3.0:1788201707.124207:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487538688, bytes: 4096 00000080:00200000:3.0:1788201707.124210:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.124214:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.124216:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487538688, 487542784) 0 0 00000080:00200000:3.0:1788201707.124218:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.124221:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119028, 119028] 00020000:00200000:3.0:1788201707.124224:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487538688 chunk: [487538688, 487542784) 487542784, 4096 00020000:00200000:3.0:1788201707.124225:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.124233:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119028, 119028] 00000008:00010000:3.0:1788201707.124237:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.124238:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.124243:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124249:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487538688 487542783) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124255:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124259:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124264:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.124268:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487538688, 487542784) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.124271:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119028 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.124274:0:17463:0:(rw26.c:898:ll_write_end()) pos 487538688, len 4096, copied 4096 00000080:00200000:3.0:1788201707.124277:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000d9f590c0[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119028] 00000080:00200000:3.0:1788201707.124279:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000bb747557 fffffc0001029 2:0 ffff93c0c13bc8c0 119028 lru 00000080:00200000:3.0:1788201707.124282:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000c97ab48c 119028: 1< 2 - - > 2< 487538688 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70613 - -> 00000080:00200000:3.0:1788201707.124284:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000d9f590c0 00000080:00200000:3.0:1788201707.124284:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.124286:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.124291:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.124302:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.124305:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.124307:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.124316:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.124318:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.124321:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.124325:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124334:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.124342:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.124345:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.124348:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.124350:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119028, 119028] 00000080:00200000:3.0:1788201707.124358:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.124360:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.124379:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.124382:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.124392:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487542784, count: 4096 00000080:00200000:3.0:1788201707.124398:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487542784, bytes: 4096 00000080:00200000:3.0:1788201707.124400:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.124402:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.124403:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487542784, 487546880) 0 0 00000080:00200000:3.0:1788201707.124405:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.124408:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119029, 119029] 00020000:00200000:3.0:1788201707.124410:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487542784 chunk: [487542784, 487546880) 487546880, 4096 00020000:00200000:3.0:1788201707.124411:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.124418:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119029, 119029] 00000008:00010000:3.0:1788201707.124421:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.124423:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.124428:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124434:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487542784 487546879) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124439:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124444:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124447:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.124452:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487542784, 487546880) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.124456:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119029 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.124459:0:17463:0:(rw26.c:898:ll_write_end()) pos 487542784, len 4096, copied 4096 00000080:00200000:3.0:1788201707.124461:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004768cced[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119029] 00000080:00200000:3.0:1788201707.124463:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fda39611 fffffc0001029 2:0 ffff93c0c13bcee0 119029 lru 00000080:00200000:3.0:1788201707.124467:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000006ae245c5 119029: 1< 2 - - > 2< 487542784 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70614 - -> 00000080:00200000:3.0:1788201707.124469:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004768cced 00000080:00200000:3.0:1788201707.124470:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.124471:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.124476:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.124484:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.124485:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.124487:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.124494:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.124497:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.124499:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.124503:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124510:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.124518:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.124521:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.124525:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.124528:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119029, 119029] 00000080:00200000:3.0:1788201707.124539:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.124541:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.124560:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.124562:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.124573:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487546880, count: 4096 00000080:00200000:3.0:1788201707.124579:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487546880, bytes: 4096 00000080:00200000:3.0:1788201707.124581:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.124583:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.124584:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487546880, 487550976) 0 0 00000080:00200000:3.0:1788201707.124586:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.124589:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119030, 119030] 00020000:00200000:3.0:1788201707.124591:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487546880 chunk: [487546880, 487550976) 487550976, 4096 00020000:00200000:3.0:1788201707.124592:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.124603:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119030, 119030] 00000008:00010000:3.0:1788201707.124608:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.124610:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.124617:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124627:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487546880 487550975) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124636:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124643:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124648:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.124656:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487546880, 487550976) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.124660:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119030 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.124664:0:17463:0:(rw26.c:898:ll_write_end()) pos 487546880, len 4096, copied 4096 00000080:00200000:3.0:1788201707.124666:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000278dfcd1[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119030] 00000080:00200000:3.0:1788201707.124669:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c3cd578f fffffc0001029 2:0 ffff93c0c13bc7e0 119030 lru 00000080:00200000:3.0:1788201707.124672:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001886b378 119030: 1< 2 - - > 2< 487546880 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70615 - -> 00000080:00200000:3.0:1788201707.124674:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000278dfcd1 00000080:00200000:3.0:1788201707.124675:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.124676:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.124681:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.124689:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.124691:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.124692:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.124698:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.124700:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.124702:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.124704:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124709:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.124714:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.124715:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.124718:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.124720:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119030, 119030] 00000080:00200000:3.0:1788201707.124728:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.124730:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.124748:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.124750:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.124760:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487550976, count: 4096 00000080:00200000:3.0:1788201707.124766:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487550976, bytes: 4096 00000080:00200000:3.0:1788201707.124768:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.124770:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.124771:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487550976, 487555072) 0 0 00000080:00200000:3.0:1788201707.124773:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.124776:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119031, 119031] 00020000:00200000:3.0:1788201707.124778:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487550976 chunk: [487550976, 487555072) 487555072, 4096 00020000:00200000:3.0:1788201707.124779:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.124788:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119031, 119031] 00000008:00010000:3.0:1788201707.124793:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.124795:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.124802:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124811:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487550976 487555071) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124819:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124827:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124849:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.124854:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487550976, 487555072) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.124858:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119031 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.124861:0:17463:0:(rw26.c:898:ll_write_end()) pos 487550976, len 4096, copied 4096 00000080:00200000:3.0:1788201707.124864:0:17463:0:(rw26.c:956:ll_write_end()) page@000000001ae44ed1[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119031] 00000080:00200000:3.0:1788201707.124865:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000028427716 fffffc0001029 2:0 ffff93c0c13bc2a0 119031 lru 00000080:00200000:3.0:1788201707.124869:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000072a29004 119031: 1< 2 - - > 2< 487550976 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70616 - -> 00000080:00200000:3.0:1788201707.124871:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000001ae44ed1 00000080:00200000:3.0:1788201707.124871:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.124873:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.124878:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.124886:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.124888:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.124889:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.124895:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.124897:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.124899:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.124901:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.124906:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.124914:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.124916:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.124918:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.124920:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119031, 119031] 00000080:00200000:3.0:1788201707.124928:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.124930:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.124948:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.124950:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.124961:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487555072, count: 4096 00000080:00200000:3.0:1788201707.124967:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487555072, bytes: 4096 00000080:00200000:3.0:1788201707.124968:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.124971:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.124972:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487555072, 487559168) 0 0 00000080:00200000:3.0:1788201707.124973:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.124976:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119032, 119032] 00020000:00200000:3.0:1788201707.124978:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487555072 chunk: [487555072, 487559168) 487559168, 4096 00020000:00200000:3.0:1788201707.124980:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.124987:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119032, 119032] 00000008:00010000:3.0:1788201707.124992:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.124994:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.125001:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125021:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487555072 487559167) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125030:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125044:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125050:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.125055:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487555072, 487559168) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.125059:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119032 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.125063:0:17463:0:(rw26.c:898:ll_write_end()) pos 487555072, len 4096, copied 4096 00000080:00200000:3.0:1788201707.125065:0:17463:0:(rw26.c:956:ll_write_end()) page@000000004c71dd75[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119032] 00000080:00200000:3.0:1788201707.125067:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000c8e4d55e fffffc0001029 2:0 ffff93c0c13bce00 119032 lru 00000080:00200000:3.0:1788201707.125071:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000992f4418 119032: 1< 2 - - > 2< 487555072 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70617 - -> 00000080:00200000:3.0:1788201707.125072:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000004c71dd75 00000080:00200000:3.0:1788201707.125073:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.125074:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.125080:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.125088:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.125089:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.125091:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.125097:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.125099:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.125100:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.125103:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125111:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.125116:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.125117:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.125120:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.125121:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119032, 119032] 00000080:00200000:3.0:1788201707.125129:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.125131:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.125149:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.125152:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.125162:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487559168, count: 4096 00000080:00200000:3.0:1788201707.125168:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487559168, bytes: 4096 00000080:00200000:3.0:1788201707.125170:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.125172:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.125173:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487559168, 487563264) 0 0 00000080:00200000:3.0:1788201707.125175:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.125178:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119033, 119033] 00020000:00200000:3.0:1788201707.125180:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487559168 chunk: [487559168, 487563264) 487563264, 4096 00020000:00200000:3.0:1788201707.125181:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.125188:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119033, 119033] 00000008:00010000:3.0:1788201707.125191:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.125192:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.125197:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125203:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487559168 487563263) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125212:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125216:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125220:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.125225:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487559168, 487563264) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.125228:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119033 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.125231:0:17463:0:(rw26.c:898:ll_write_end()) pos 487559168, len 4096, copied 4096 00000080:00200000:3.0:1788201707.125234:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000c72b5f44[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119033] 00000080:00200000:3.0:1788201707.125235:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000017a94b36 fffffc0001029 2:0 ffff93c0c13bc1c0 119033 lru 00000080:00200000:3.0:1788201707.125239:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000001567f8db 119033: 1< 2 - - > 2< 487559168 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70618 - -> 00000080:00200000:3.0:1788201707.125240:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000c72b5f44 00000080:00200000:3.0:1788201707.125241:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.125242:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.125248:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.125255:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.125257:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.125258:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.125264:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.125266:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.125267:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.125270:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125275:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.125279:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.125281:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.125283:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.125285:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119033, 119033] 00000080:00200000:3.0:1788201707.125292:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.125295:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.125321:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.125324:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.125341:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487563264, count: 4096 00000080:00200000:3.0:1788201707.125350:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487563264, bytes: 4096 00000080:00200000:3.0:1788201707.125352:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.125354:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.125355:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487563264, 487567360) 0 0 00000080:00200000:3.0:1788201707.125358:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.125363:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119034, 119034] 00020000:00200000:3.0:1788201707.125365:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487563264 chunk: [487563264, 487567360) 487567360, 4096 00020000:00200000:3.0:1788201707.125367:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.125380:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119034, 119034] 00000008:00010000:3.0:1788201707.125385:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.125387:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.125394:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125406:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487563264 487567359) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125415:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125421:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125427:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.125433:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487563264, 487567360) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.125438:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119034 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.125443:0:17463:0:(rw26.c:898:ll_write_end()) pos 487563264, len 4096, copied 4096 00000080:00200000:3.0:1788201707.125447:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000045a43feb[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119034] 00000080:00200000:3.0:1788201707.125450:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000005897a454 fffffc0001029 2:0 ffff93c0c13bc620 119034 lru 00000080:00200000:3.0:1788201707.125455:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000a6020175 119034: 1< 2 - - > 2< 487563264 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70619 - -> 00000080:00200000:3.0:1788201707.125456:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000045a43feb 00000080:00200000:3.0:1788201707.125457:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.125459:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.125467:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.125478:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.125481:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.125483:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.125492:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.125494:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.125496:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.125500:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125508:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.125515:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.125517:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.125521:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.125523:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119034, 119034] 00000080:00200000:3.0:1788201707.125534:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.125538:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.125564:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.125568:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.125583:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487567360, count: 4096 00000080:00200000:3.0:1788201707.125593:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487567360, bytes: 4096 00000080:00200000:3.0:1788201707.125595:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.125599:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.125601:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487567360, 487571456) 0 0 00000080:00200000:3.0:1788201707.125603:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.125609:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119035, 119035] 00020000:00200000:3.0:1788201707.125612:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487567360 chunk: [487567360, 487571456) 487571456, 4096 00020000:00200000:3.0:1788201707.125614:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.125626:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119035, 119035] 00000008:00010000:3.0:1788201707.125631:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.125633:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.125641:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125651:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487567360 487571455) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125659:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125667:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125674:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.125680:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487567360, 487571456) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.125684:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119035 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.125688:0:17463:0:(rw26.c:898:ll_write_end()) pos 487567360, len 4096, copied 4096 00000080:00200000:3.0:1788201707.125691:0:17463:0:(rw26.c:956:ll_write_end()) page@000000008ea031bb[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119035] 00000080:00200000:3.0:1788201707.125693:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000001b2410ef fffffc0001029 2:0 ffff93c0c13bcd20 119035 lru 00000080:00200000:3.0:1788201707.125697:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000e87b0f40 119035: 1< 2 - - > 2< 487567360 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70620 - -> 00000080:00200000:3.0:1788201707.125698:0:17463:0:(rw26.c:956:ll_write_end()) end page@000000008ea031bb 00000080:00200000:3.0:1788201707.125699:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.125700:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.125706:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.125714:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.125716:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.125717:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.125723:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.125725:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.125727:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.125729:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125734:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.125739:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.125741:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.125743:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.125745:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119035, 119035] 00000080:00200000:3.0:1788201707.125752:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.125754:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.125773:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.125775:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.125787:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487571456, count: 4096 00000080:00200000:3.0:1788201707.125797:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487571456, bytes: 4096 00000080:00200000:3.0:1788201707.125799:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.125803:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.125804:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487571456, 487575552) 0 0 00000080:00200000:3.0:1788201707.125807:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.125812:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119036, 119036] 00020000:00200000:3.0:1788201707.125815:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487571456 chunk: [487571456, 487575552) 487575552, 4096 00020000:00200000:3.0:1788201707.125817:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.125845:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119036, 119036] 00000008:00010000:3.0:1788201707.125851:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.125853:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.125860:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125868:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487571456 487575551) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125874:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125878:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125882:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.125888:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487571456, 487575552) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.125891:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119036 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.125895:0:17463:0:(rw26.c:898:ll_write_end()) pos 487571456, len 4096, copied 4096 00000080:00200000:3.0:1788201707.125897:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000cca1f6da[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119036] 00000080:00200000:3.0:1788201707.125899:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000654a69d2 fffffc0001029 2:0 ffff93c0c13bc9a0 119036 lru 00000080:00200000:3.0:1788201707.125904:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000bf1c229f 119036: 1< 2 - - > 2< 487571456 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70621 - -> 00000080:00200000:3.0:1788201707.125906:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000cca1f6da 00000080:00200000:3.0:1788201707.125907:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.125909:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.125917:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.125933:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.125936:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.125938:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.125947:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.125949:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.125951:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.125956:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.125963:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.125970:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.125973:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.125976:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.125978:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119036, 119036] 00000080:00200000:3.0:1788201707.125989:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.125992:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.126033:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.126036:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.126052:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487575552, count: 4096 00000080:00200000:3.0:1788201707.126062:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487575552, bytes: 4096 00000080:00200000:3.0:1788201707.126064:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.126068:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.126069:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487575552, 487579648) 0 0 00000080:00200000:3.0:1788201707.126071:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.126075:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119037, 119037] 00020000:00200000:3.0:1788201707.126079:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487575552 chunk: [487575552, 487579648) 487579648, 4096 00020000:00200000:3.0:1788201707.126081:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.126092:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119037, 119037] 00000008:00010000:3.0:1788201707.126096:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.126099:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.126106:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126115:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487575552 487579647) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126124:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126133:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126140:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.126147:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487575552, 487579648) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.126151:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119037 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.126156:0:17463:0:(rw26.c:898:ll_write_end()) pos 487575552, len 4096, copied 4096 00000080:00200000:3.0:1788201707.126160:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000071bb5b25[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119037] 00000080:00200000:3.0:1788201707.126162:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000a268f851 fffffc0001029 2:0 ffff93c0c13bd2a0 119037 lru 00000080:00200000:3.0:1788201707.126168:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000aff70a33 119037: 1< 2 - - > 2< 487575552 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70622 - -> 00000080:00200000:3.0:1788201707.126170:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000071bb5b25 00000080:00200000:3.0:1788201707.126171:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.126173:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.126181:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.126193:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.126196:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.126198:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.126206:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.126209:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.126211:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.126215:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126224:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.126231:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.126234:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.126237:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.126240:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119037, 119037] 00000080:00200000:3.0:1788201707.126250:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.126253:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.126278:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.126281:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.126295:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487579648, count: 4096 00000080:00200000:3.0:1788201707.126305:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487579648, bytes: 4096 00000080:00200000:3.0:1788201707.126308:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.126311:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.126313:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487579648, 487583744) 0 0 00000080:00200000:3.0:1788201707.126316:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.126320:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119038, 119038] 00020000:00200000:3.0:1788201707.126323:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487579648 chunk: [487579648, 487583744) 487583744, 4096 00020000:00200000:3.0:1788201707.126325:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.126335:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119038, 119038] 00000008:00010000:3.0:1788201707.126340:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.126342:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.126349:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126358:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487579648 487583743) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126367:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126375:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126382:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.126389:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487579648, 487583744) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.126393:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119038 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.126398:0:17463:0:(rw26.c:898:ll_write_end()) pos 487579648, len 4096, copied 4096 00000080:00200000:3.0:1788201707.126402:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000031506669[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119038] 00000080:00200000:3.0:1788201707.126405:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000f335899f fffffc0001029 2:0 ffff93c0c13bde00 119038 lru 00000080:00200000:3.0:1788201707.126411:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@0000000007d826cc 119038: 1< 2 - - > 2< 487579648 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70623 - -> 00000080:00200000:3.0:1788201707.126414:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000031506669 00000080:00200000:3.0:1788201707.126416:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.126417:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.126424:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.126435:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.126437:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.126440:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.126447:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.126450:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.126452:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.126456:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126464:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.126472:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.126474:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.126477:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.126480:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119038, 119038] 00000080:00200000:3.0:1788201707.126490:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.126494:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.126517:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.126520:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.126534:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487583744, count: 4096 00000080:00200000:3.0:1788201707.126544:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487583744, bytes: 4096 00000080:00200000:3.0:1788201707.126546:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.126550:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.126551:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487583744, 487587840) 0 0 00000080:00200000:3.0:1788201707.126554:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.126558:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119039, 119039] 00020000:00200000:3.0:1788201707.126561:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487583744 chunk: [487583744, 487587840) 487587840, 4096 00020000:00200000:3.0:1788201707.126563:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.126572:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119039, 119039] 00000008:00010000:3.0:1788201707.126577:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.126578:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.126585:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126594:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487583744 487587839) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126603:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126611:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126619:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.126626:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487583744, 487587840) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.126630:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119039 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.126634:0:17463:0:(rw26.c:898:ll_write_end()) pos 487583744, len 4096, copied 4096 00000080:00200000:3.0:1788201707.126638:0:17463:0:(rw26.c:956:ll_write_end()) page@00000000e4889f77[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119039] 00000080:00200000:3.0:1788201707.126641:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @000000003de91fd4 fffffc0001029 2:0 ffff93c0c13bd1c0 119039 lru 00000080:00200000:3.0:1788201707.126647:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000ac4bc998 119039: 1< 2 - - > 2< 487583744 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70624 - -> 00000080:00200000:3.0:1788201707.126649:0:17463:0:(rw26.c:956:ll_write_end()) end page@00000000e4889f77 00000080:00200000:3.0:1788201707.126650:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.126652:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.126659:0:17463:0:(osc_cache.c:845:osc_extent_find()) ### extent: 0000000030f139e6 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.126669:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.126671:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.126673:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.126681:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.126683:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.126685:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.126689:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126698:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.126706:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.126708:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.126711:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.126714:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119039, 119039] 00000080:00200000:3.0:1788201707.126724:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.126728:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.126751:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.126755:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.126770:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487587840, count: 4096 00000080:00200000:3.0:1788201707.126779:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487587840, bytes: 4096 00000080:00200000:3.0:1788201707.126781:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.126785:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.126786:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487587840, 487591936) 0 0 00000080:00200000:3.0:1788201707.126789:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.126793:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119040, 119040] 00020000:00200000:3.0:1788201707.126796:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487587840 chunk: [487587840, 487591936) 487591936, 4096 00020000:00200000:3.0:1788201707.126798:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.126808:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119040, 119040] 00000008:00010000:3.0:1788201707.126812:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.126814:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.126821:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126845:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487587840 487591935) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 287/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126855:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126863:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126871:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.126878:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487587840, 487591936) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.126882:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119040 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.126887:0:17463:0:(rw26.c:898:ll_write_end()) pos 487587840, len 4096, copied 4096 00000080:00200000:3.0:1788201707.126891:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000080bd35f3[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119040] 00000080:00200000:3.0:1788201707.126894:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @00000000fd71145a fffffc0001029 2:0 ffff93c0c4405b60 119040 lru 00000080:00200000:3.0:1788201707.126900:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@000000004a18567a 119040: 1< 2 - - > 2< 487587840 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 70625 - -> 00000080:00200000:3.0:1788201707.126903:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000080bd35f3 00000080:00200000:3.0:1788201707.126904:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.126906:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1788201707.126914:0:17463:0:(osc_cache.c:857:osc_extent_find()) ### extent: 00000000d448c6d2 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.126924:0:17463:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff93c1d26a4478, count 1, dirtied 1 00000080:00200000:3.0:1788201707.126927:0:17463:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1788201707.126929:0:17463:0:(vvp_io.c:1358:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1788201707.126936:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_release():245 00000020:00010000:3.0:1788201707.126939:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788201707.126941:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff93c1d2f7f330 (ffff93c1d864fab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788201707.126946:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.126954:0:17463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,0 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1788201707.126962:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1c83e3b78 (ffff93c1d864f008/0) at cl_lock_fini():72 00000020:00010000:3.0:1788201707.126964:0:17463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff93c1d2f7f330 (ffff93c1d864f008/1) at cl_lock_fini():72 00000080:00200000:3.0:1788201707.126967:0:17463:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1788201707.126970:0:17463:0:(file.c:2205:ll_file_io_generic()) Range unlock [119040, 119040] 00000080:00200000:3.0:1788201707.126980:0:17463:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201707.126984:0:17463:0:(file.c:2238:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1788201707.127021:0:17463:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x10:0x0] updating i_size 1073741824 i_blocks 2097321 00000080:00200000:3.0:1788201707.127025:0:17463:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1788201707.127040:0:17463:0:(file.c:2638:do_file_write_iter()) START file f5.replay-ost-single:[0x200000402:0x10:0x0], ppos: 487591936, count: 4096 00000080:00200000:3.0:1788201707.127050:0:17463:0:(file.c:2090:ll_file_io_generic()) f5.replay-ost-single: write ppos: 487591936, bytes: 4096 00000080:00200000:3.0:1788201707.127052:0:17463:0:(file.c:1848:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1788201707.127056:0:17463:0:(cl_io.c:214:cl_io_rw_init()) header@00000000578309b5[0x110, 262145, [0x200000402:0x10:0x0] hash] 00000020:00200000:3.0:1788201707.127057:0:17463:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [487591936, 487596032) 0 0 00000080:00200000:3.0:1788201707.127060:0:17463:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788201707.127064:0:17463:0:(file.c:2168:ll_file_io_generic()) Range lock [119041, 119041] 00020000:00200000:3.0:1788201707.127067:0:17463:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 487591936 chunk: [487591936, 487596032) 487596032, 4096 00020000:00200000:3.0:1788201707.127069:0:17463:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201707.127080:0:17463:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [119041, 119041] 00000008:00010000:3.0:1788201707.127084:0:17463:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788201707.127085:0:17463:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff93c1d2f7f330, osc lock ffff93c1d1b98be0, flags 0x0 00010000:00010000:3.0:1788201707.127092:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.127101:0:17463:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487591936 487596031) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 288/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.127110:0:17463:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 292/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.127118:0:17463:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 292/0,2 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201707.127125:0:17463:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 291/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1788201707.127133:0:17463:0:(vvp_io.c:1292:vvp_io_write_start()) f5.replay-ost-single: write [487591936, 487596032) append 0 append lockpos 0 00000080:00200000:3.0:1788201707.127137:0:17463:0:(rw26.c:704:ll_write_begin()) Writing 119041 of 0 to 4096 bytes 00000080:00200000:3.0:1788201707.127142:0:17463:0:(rw26.c:898:ll_write_end()) pos 487591936, len 4096, copied 4096 00000080:00200000:3.0:1788201707.127146:0:17463:0:(rw26.c:956:ll_write_end()) page@0000000026599f17[3 0000000075f4bf38 2 1 00000000863bc12b 0 0 119041] 00000080:00200000:3.0:1788201707.127149:0:17463:0:(rw26.c:956:ll_write_end()) vmpage @0000000048350093 fffffc0001029 2:0 ffff93c0c4405c40 119041 lru 00000080:00200000:3.0:1788201707.127155:0:17463:0:(rw26.c:956:ll_write_end()) osc-page@00000000673a760e 119041: 1< 2 - - > 2< 487591936 0 4096 0x0 0x40420 | 00000000795b15c4 00000000f3c0a95d > 3< 0 0 > 4< 0 9 8 0 - | + - + - > 5< + - + - | 0 - | 70626 - -> 00000080:00200000:3.0:1788201707.127157:0:17463:0:(rw26.c:956:ll_write_end()) end page@0000000026599f17 00000080:00200000:3.0:1788201707.127158:0:17463:0:(rw26.c:956:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788201707.127160:0:17463:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000100:00000400:1.0:1788201707.245955:0:2341:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201691/real 1788201691] req@ffff93c1c8485f80 x1875064559762176/t0(0) o4->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:6/4 lens 488/448 e 0 to 1 dl 1788201707 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00000400:3.0:1788201707.245955:0:2344:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201691/real 1788201691] req@ffff93c1c53e0700 x1875064559762048/t0(0) o4->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:6/4 lens 488/448 e 0 to 1 dl 1788201707 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00000400:0.0:1788201707.245960:0:2342:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201691/real 1788201691] req@ffff93c1c5036680 x1875064559761920/t0(0) o400->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201707 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201707.246030:0:2341:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:1.0:1788201707.246035:0:2341:0:(import.c:182:ptlrpc_set_import_discon()) lustre-OST0000-osc-ffff93c1d85e6000: Connection to lustre-OST0000 (at 192.168.201.156@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00100000:0.0:1788201707.246041:0:2342:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff93c1c5036680 x1875064559761920/t0(0) o400->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201707 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788201707.246056:0:2342:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:0.0:1788201707.246060:0:2342:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff93c1d85e6000: Connection to lustre-MDT0000 (at 192.168.201.156@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:0.0:1788201707.246067:0:2342:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00000400:0.0:1788201707.246092:0:2342:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201691/real 1788201691] req@ffff93c1c5035c00 x1875064559761792/t0(0) o400->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 224/224 e 0 to 1 dl 1788201707 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788201707.246103:0:2342:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff93c1c5035c00 x1875064559761792/t0(0) o400->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 224/224 e 0 to 1 dl 1788201707 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788201707.246108:0:2342:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1788201707.246110:0:2342:0:(import.c:186:ptlrpc_set_import_discon()) MGC192.168.201.156@tcp: Connection to MGS (at 192.168.201.156@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1788201707.246116:0:2342:0:(import.c:192:ptlrpc_set_import_discon()) import MGS@192.168.201.156@tcp for MGC192.168.201.156@tcp not replayable, auto-deactivating 00000100:00080000:0.0:1788201707.246118:0:2342:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:0.0:1788201707.246121:0:2342:0:(client.c:3576:ptlrpc_abort_inflight()) @@@ inflight req@ffff93c1c5035c00 x1875064559761792/t0(0) o400->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 224/224 e 0 to 1 dl 1788201707 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788201707.246127:0:2342:0:(client.c:3576:ptlrpc_abort_inflight()) @@@ inflight req@ffff93c0e62bb480 x1875064559762560/t0(0) o400->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 224/224 e 0 to 0 dl 1788201713 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788201707.246144:0:2342:0:(client.c:3576:ptlrpc_abort_inflight()) @@@ inflight req@ffff93c1c5034a80 x1875064559762944/t0(0) o400->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 224/224 e 0 to 0 dl 1788201718 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:0.0:1788201707.246149:0:2342:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1788201707.246151:0:2342:0:(pinger.c:193:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:0.0:1788201707.246152:0:2342:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808002 00000100:00080000:0.0:1788201707.246153:0:2342:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000400:0.0:1788201707.246158:0:2342:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201691/real 1788201691] req@ffff93c1c8484a80 x1875064559761536/t0(0) o4->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:6/4 lens 488/448 e 0 to 1 dl 1788201707 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1788201707.246197:0:2343:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:961d4966-9b2e-4bca-bd35-8d9493e230e8:2343:1875064559762560:192.168.201.156@tcp:400:kworker.0 00000100:00080000:2.0:1788201707.246250:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201707.246255:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201707.246257:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201707.246262:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 306 00000100:00080000:2.0:1788201707.246273:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201707.246301:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559763328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201707.246320:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201707.246330:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201707.246332:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201707.246335:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 306 00000100:00080000:2.0:1788201707.246340:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201707.246347:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201707.246359:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559763456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201707.264452:0:2341:0:(import.c:418:ptlrpc_pinger_force()) lustre-OST0000_UUID: waking up pinger s:DISCONN 00000100:00080000:0.0:1788201707.264467:0:2342:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff93c1d3241800 already not connected (conn 31, was 31): DISCONN 00000100:00080000:2.0:1788201707.264474:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201707.264479:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201707.264482:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201707.264496:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 595 00000100:00080000:2.0:1788201707.264508:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00080000:3.0:1788201707.264518:0:2344:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff93c1d3241800 already not connected (conn 32, was 31): CONNECTING 00000100:00100000:2.0:1788201707.264528:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559763328:192.168.201.156@tcp:250:kworker.0 00000100:00000400:3.0:1788201707.264534:0:2344:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201691/real 1788201691] req@ffff93c0e62ba300 x1875064559761152/t0(0) o4->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:6/4 lens 488/448 e 0 to 1 dl 1788201707 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00080000:3.0:1788201707.264561:0:2344:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff93c1d3241800 already not connected (conn 32, was 31): CONNECTING 00000100:00000400:1.0:1788201707.264569:0:2341:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201691/real 1788201691] req@ffff93c1c5036a00 x1875064559761664/t0(0) o4->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:6/4 lens 488/448 e 0 to 1 dl 1788201707 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00080000:1.0:1788201707.264596:0:2341:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff93c1d3241800 already not connected (conn 32, was 31): CONNECTING 00000100:00000400:1.0:1788201707.264605:0:2341:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201691/real 1788201691] req@ffff93c1c8484380 x1875064559761408/t0(0) o4->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:6/4 lens 488/448 e 0 to 1 dl 1788201707 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1788201707.264613:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559763456:192.168.201.156@tcp:38:kworker.0 00000008:00100000:3.0:1788201707.264614:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00080000:1.0:1788201707.264624:0:2341:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff93c1d3241800 already not connected (conn 32, was 31): CONNECTING 00000100:00000400:1.0:1788201707.264633:0:2341:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201691/real 1788201691] req@ffff93c1c8487800 x1875064559761280/t0(0) o4->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:6/4 lens 488/448 e 0 to 1 dl 1788201707 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00100000:3.0:1788201707.264647:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c53e1f80 x1875064559763584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201707.264651:0:2341:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff93c1d3241800 already not connected (conn 32, was 31): CONNECTING 00000100:00100000:1.0:1788201707.264671:0:2341:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:961d4966-9b2e-4bca-bd35-8d9493e230e8:2341:1875064559762944:192.168.201.156@tcp:400:kworker.0 00000100:00100000:3.0:1788201707.264709:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c53e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559763584:192.168.201.156@tcp:8:kworker.0 00000100:00100000:1.0:1788201707.264728:0:2342:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:986744c8-ee81-4a47-8d91-97f81ede1f3e:2342:1875064559761920:192.168.201.156@tcp:400:kworker.0 00000100:00100000:1.0:1788201707.264736:0:2342:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:961d4966-9b2e-4bca-bd35-8d9493e230e8:2342:1875064559761792:192.168.201.156@tcp:400:kworker.0 00010000:00010000:0.0:1788201707.949975:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1788201708.269968:0:2342:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201692/real 1788201692] req@ffff93c1c8484e00 x1875064559762432/t0(0) o4->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:6/4 lens 488/448 e 0 to 1 dl 1788201708 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00080000:1.0:1788201708.282259:0:2342:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff93c1d3241800 already not connected (conn 32, was 31): CONNECTING 00000100:00000400:1.0:1788201708.282278:0:2342:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201692/real 1788201692] req@ffff93c1c5036300 x1875064559762304/t0(0) o4->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:6/4 lens 488/448 e 0 to 1 dl 1788201708 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00080000:1.0:1788201708.282296:0:2342:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff93c1d3241800 already not connected (conn 32, was 31): CONNECTING 00010000:00010000:2.0:1788201708.973999:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201709.997926:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788201710.830029:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201710.830042:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201710.830152:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201710.830183:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201710.830188:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201710.830219:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788201711.021928:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201712.045919:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201712.366051:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201712.366059:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1788201712.366062:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201712.366064:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1788201712.366066:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201712.366069:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1788201713.069944:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1788201713.134002:0:2343:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201697/real 1788201697] req@ffff93c0e62b8380 x1875064559762816/t0(0) o400->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 224/224 e 0 to 1 dl 1788201713 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201713.145863:0:2343:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff93c0e62b8380 x1875064559762816/t0(0) o400->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 224/224 e 0 to 1 dl 1788201713 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201713.145875:0:2343:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff93c1d3241800 already not connected (conn 32, was 31): CONNECTING 00000100:00100000:2.0:1788201713.145894:0:2343:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:986744c8-ee81-4a47-8d91-97f81ede1f3e:2343:1875064559762816:192.168.201.156@tcp:400:kworker.0 00000100:00000400:2.0:1788201713.389956:0:2344:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201697/real 1788201697] req@ffff93c0e62b8700 x1875064559762688/t0(0) o400->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201713 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201713.389986:0:2344:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff93c0e62b8700 x1875064559762688/t0(0) o400->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201713 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201713.389996:0:2344:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff93c1c4d85000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:2.0:1788201713.390007:0:2344:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:986744c8-ee81-4a47-8d91-97f81ede1f3e:2344:1875064559762688:192.168.201.156@tcp:400:kworker.0 00010000:00010000:0.0:1788201714.093983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788201715.117998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201715.949971:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201715.949983:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201715.950061:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201715.950085:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201715.950088:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201715.950107:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201716.141968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788201717.166051:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788201717.485961:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201707/real 1788201707] req@ffff93c1c53e1f80 x1875064559763584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201717 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201717.485962:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201717.485978:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788201717.485982:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201717.485993:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788201717.485999:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201717.486004:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1788201717.486028:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c53e1f80 x1875064559763584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201717 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201717.486042:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201707/real 1788201707] req@ffff93c0e62bb480 x1875064559763328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201717 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201717.486056:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559763328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201717 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201717.486067:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201707/real 1788201707] req@ffff93c0e62baa00 x1875064559763456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201717 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201717.486078:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559763456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201717 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201717.486092:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201717.486109:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201717.486112:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c53e1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559763584:192.168.201.156@tcp:8:kworker.0 00000100:00080000:3.0:1788201717.486118:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201717.486125:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201717.486128:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559763328:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788201717.486132:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201717.486136:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201717.486139:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201717.486142:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201717.486145:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201717.486154:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559763456:192.168.201.156@tcp:38:kworker.0 00000100:00080000:2.0:1788201717.486156:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 843 00000100:00080000:2.0:1788201717.486170:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1788201717.486177:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201717.486208:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559763712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201717.486229:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201717.486231:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201717.486232:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201717.486235:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 843 00000100:00080000:2.0:1788201717.486238:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1788201717.486240:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201717.486246:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201717.486253:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559763840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201717.486258:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201717.486260:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201717.486261:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201717.486268:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 843 00000100:00080000:2.0:1788201717.486272:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1788201717.486274:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201717.486278:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201717.486285:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559763968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201717.486288:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559763712:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201717.486401:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559763840:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201717.486445:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559763968:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201718.189985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1788201718.509942:0:2341:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201702/real 1788201702] req@ffff93c1c5037800 x1875064559763200/t0(0) o400->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 224/224 e 0 to 1 dl 1788201718 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:1.0:1788201718.509942:0:2342:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201702/real 1788201702] req@ffff93c1c5037100 x1875064559763072/t0(0) o400->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201718 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788201718.509992:0:2342:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff93c1c5037100 x1875064559763072/t0(0) o400->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 224/224 e 0 to 1 dl 1788201718 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788201718.509999:0:2342:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff93c1c4d85000 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:1.0:1788201718.510044:0:2342:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:986744c8-ee81-4a47-8d91-97f81ede1f3e:2342:1875064559763072:192.168.201.156@tcp:400:kworker.0 00000100:00100000:0.0:1788201718.522698:0:2341:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff93c1c5037800 x1875064559763200/t0(0) o400->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 224/224 e 0 to 1 dl 1788201718 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788201718.522712:0:2341:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff93c1d3241800 already not connected (conn 33, was 31): CONNECTING 00000100:00100000:0.0:1788201718.522731:0:2341:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:986744c8-ee81-4a47-8d91-97f81ede1f3e:2341:1875064559763200:192.168.201.156@tcp:400:kworker.0 00010000:00010000:2.0:1788201719.213976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201720.237991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201721.069998:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201721.070018:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201721.070141:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201721.070161:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201721.070164:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201721.070181:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788201721.262020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201722.285983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201722.605957:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201722.605975:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788201722.605979:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201722.605982:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788201722.605985:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201722.605988:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:3.0:1788201723.309996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201724.333968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201725.357952:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201726.190039:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201726.190072:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201726.190191:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201726.190219:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201726.190225:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201726.190253:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201726.381966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201727.405974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201727.725943:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201727.725954:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788201727.725958:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201727.725961:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788201727.725965:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201727.725969:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1788201728.429968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201729.453987:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201730.478024:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201731.310012:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201731.310026:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201731.310106:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201731.310124:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201731.310127:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201731.310145:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788201731.501948:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201732.525964:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201732.845945:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788201732.845945:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201717/real 1788201717] req@ffff93c0e62b8000 x1875064559763968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201732 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201732.845961:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788201732.845965:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201732.845976:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788201732.845982:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201732.845989:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1788201732.846011:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559763968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201732 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201732.846024:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201717/real 1788201717] req@ffff93c0e62b8700 x1875064559763712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201732 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201732.846035:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559763712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201732 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201732.846043:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201717/real 1788201717] req@ffff93c0e62b8380 x1875064559763840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201732 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201732.846053:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559763840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201732 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201732.846082:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201732.846102:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201732.846107:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559763968:192.168.201.156@tcp:8:kworker.0 00000100:00080000:3.0:1788201732.846114:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201732.846121:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00080000:2.0:1788201732.846127:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788201732.846127:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559763712:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788201732.846141:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201732.846146:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201732.846156:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201732.846161:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 853 00000100:00080000:2.0:1788201732.846174:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1788201732.846176:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201732.846183:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559763840:192.168.201.156@tcp:38:kworker.0 00000100:00080000:2.0:1788201732.846188:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201732.846239:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559764096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201732.846260:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201732.846263:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201732.846265:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201732.846268:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 853 00000100:00080000:2.0:1788201732.846271:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1788201732.846276:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201732.846287:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201732.846301:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559764096:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201732.846302:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559764224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201732.846325:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201732.846327:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201732.846328:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201732.846331:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 853 00000100:00080000:2.0:1788201732.846333:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1788201732.846336:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201732.846339:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201732.846348:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559764352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201732.846452:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559764224:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201732.847702:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559764352:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788201733.550008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201734.573946:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201735.598076:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788201736.430026:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201736.430041:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201736.430181:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201736.430221:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201736.430227:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201736.430262:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201736.621964:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201737.646061:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788201737.965974:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201737.965988:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788201737.965991:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201737.965998:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788201737.966002:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201737.966006:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1788201738.669981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201739.693987:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201740.717967:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788201741.550012:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201741.550032:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201741.550167:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201741.550196:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201741.550200:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201741.550226:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788201741.741961:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201742.766042:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788201743.085897:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201743.085906:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788201743.085909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201743.085912:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788201743.085914:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201743.085916:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:3.0:1788201743.789996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201744.813993:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201745.838032:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788201746.670012:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201746.670031:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201746.670155:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201746.670179:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201746.670182:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201746.670200:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201746.861966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201747.886082:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788201748.205913:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201748.205927:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788201748.205931:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201748.205934:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1788201748.205937:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201748.205939:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000800:00000100:1.1F:1788201748.397998:0:2332:0:(socklnd_cb.c:2287:ksocknal_find_timed_out_conn()) Timeout sending data to 12345-192.168.201.156@tcp (192.168.201.156:988) the network or that node may be down. 00000100:00020000:1.0:1788201748.398171:0:2332:0:(events.c:192:client_bulk_callback()) event type 1, status -5, req ffff93c0e62ba300 desc ffff93c0e0fc3400 mbits 1875064559761152 00000400:02000040:3.0:1788201748.461957:0:2338:0:(lib-move.c:3830:lnet_print_recovery_list()) 1 local NIs in recovery (showing 1): 192.168.201.56@tcp 00010000:00010000:0.0:1788201748.909966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201749.934019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201750.957984:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201751.789997:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201751.790011:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201751.791253:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201751.791282:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201751.791286:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201751.791310:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788201751.982005:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788201752.301959:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201732/real 1788201732] req@ffff93c0e62bbb80 x1875064559764224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201752 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201752.302002:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559764224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201752 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201752.302024:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201732/real 1788201732] req@ffff93c0e62b9c00 x1875064559764352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201752 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201752.302040:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559764352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201752 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201752.302052:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788201732/real 1788201732] req@ffff93c0e62ba680 x1875064559764096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201752 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201752.302066:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559764096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201752 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201752.302085:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201752.302112:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201752.302116:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559764224:192.168.201.156@tcp:38:kworker.0 00000100:00080000:3.0:1788201752.302124:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201752.302128:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201752.302131:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559764352:192.168.201.156@tcp:8:kworker.0 00000100:00080000:3.0:1788201752.302135:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201752.302138:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201752.302141:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559764096:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788201752.302159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201752.302165:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201752.302168:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201752.302175:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 868 00000100:00080000:2.0:1788201752.302187:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1788201752.302195:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201752.302249:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559764480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201752.302268:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201752.302270:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201752.302272:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201752.302275:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 868 00000100:00080000:2.0:1788201752.302278:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1788201752.302281:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201752.302291:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201752.302301:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559764608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201752.302306:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559764480:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788201752.302320:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201752.302322:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201752.302323:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201752.302327:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 868 00000100:00080000:2.0:1788201752.302329:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1788201752.302331:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201752.302335:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201752.302345:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559764736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201752.303676:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559764608:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201752.303731:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559764736:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201753.006026:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788201753.198226:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.156:988 00000400:00000100:0.0:1788201753.198274:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788201753.198289:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788201753.198335:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788201753.198345:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788201753.198385:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201752/real 1788201753] req@ffff93c0e62bb100 x1875064559764608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201753.198410:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559764608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201753.198422:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201753.198428:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201753.198432:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559764608:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201753.198446:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201752/real 1788201753] req@ffff93c0e62b9f80 x1875064559764736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201753.198462:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559764736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201753.198472:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201753.198476:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201753.198480:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559764736:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788201753.198488:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201752/real 1788201753] req@ffff93c0e62bad80 x1875064559764480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201753.198501:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559764480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201753.198510:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201753.198514:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201753.198517:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559764480:192.168.201.156@tcp:250:kworker.0 00010000:00010000:3.0:1788201754.029999:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201755.053984:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201756.078018:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201756.911206:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201756.911221:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201756.911303:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201756.911328:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201756.911330:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201756.911347:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201757.101993:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201757.421960:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201757.421968:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201757.421973:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201757.421981:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 888 00000100:00080000:2.0:1788201757.421990:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201757.422004:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201757.422040:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559764864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201757.422068:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201757.422071:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201757.422072:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201757.422083:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 888 00000100:00080000:2.0:1788201757.422086:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201757.422090:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201757.422102:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201757.422112:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559764992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201757.422124:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788201757.422126:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559764864:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788201757.422128:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201757.422129:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201757.422151:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 888 00000100:00080000:2.0:1788201757.422154:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201757.422158:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201757.422163:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201757.422173:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559765120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201757.422301:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559764992:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201757.422446:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559765120:192.168.201.156@tcp:8:kworker.0 00000008:00010000:3.0:1788201757.933999:0:17463:0:(osc_cache.c:3010:osc_queue_sync_pages()) ### no dlm lock: extent: ffff93c0e613ea80 00000008:00010000:3.0:1788201757.934059:0:10992:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93c0e613ea80 00000008:00010000:3.0:1788201757.934066:0:10992:0:(osc_cache.c:2022:try_to_add_extent_for_io()) ### extent: 0000000082ca60c7 ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 289/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1788201757.934103:0:10992:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (487591936 487596031) ns: lustre-OST0000-osc-ffff93c1d85e6000 lock: 00000000ac023428/0x390884812bd99b58 lrc: 290/0,1 mode: PW/PW res: [0x240000400:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xa81c4c9a2f3cb722 expref: -99 pid: 17463 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00100000:3.0:1788201757.934204:0:10992:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff93c1c53e1f80 - object 0x240000400:17 offset 487591936<>487596032 00000100:00080000:3.0:1788201757.934281:0:2344:0:(client.c:1832:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff93c1c53e1f80 x1875064559765248/t0(0) o4->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:6/4 lens 4584/448 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00010000:00010000:3.0:1788201758.126057:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201759.149989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788201759.342126:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.156:988 00000400:00000100:1.0:1788201759.342164:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788201759.342178:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788201759.342214:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788201759.342228:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788201759.342252:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201757/real 1788201759] req@ffff93c0e62b9500 x1875064559764992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201759.342280:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559764992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201759.342298:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201759.342304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201759.342308:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559764992:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201759.342317:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201757/real 1788201759] req@ffff93c0e62bb800 x1875064559765120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201759.342329:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559765120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201759.342338:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201759.342347:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201759.342352:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559765120:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788201759.342359:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201757/real 1788201759] req@ffff93c0e62b8a80 x1875064559764864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201759.342382:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559764864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201759.342390:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201759.342393:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201759.342396:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559764864:192.168.201.156@tcp:250:kworker.0 00010000:00010000:3.0:1788201760.175143:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201761.198000:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201762.030043:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201762.030084:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201762.030229:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201762.030304:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201762.030325:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201762.030395:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788201762.222022:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201762.541948:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201762.541958:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201762.541962:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201762.541977:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 893 00000100:00080000:2.0:1788201762.541988:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201762.541997:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201762.542038:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559765376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201762.542085:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201762.542089:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201762.542091:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201762.542095:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 893 00000100:00080000:2.0:1788201762.542100:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201762.542104:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201762.542114:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201762.542131:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559765376:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201762.542132:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559765504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201762.542147:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201762.542150:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201762.542151:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201762.542154:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 893 00000100:00080000:2.0:1788201762.542157:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201762.542160:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201762.542165:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201762.542175:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559765632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201762.542232:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559765504:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201762.542253:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559765632:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201763.245993:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201764.269991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201765.293972:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788201765.486206:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.156:988 00000400:00000100:1.0:1788201765.486290:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788201765.486311:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788201765.486350:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788201765.486360:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788201765.486402:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201762/real 1788201765] req@ffff93c0e62bb800 x1875064559765504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201765.486430:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559765504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201765.486443:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201765.486448:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201765.486453:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559765504:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201765.486461:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201762/real 1788201765] req@ffff93c0e62b9500 x1875064559765632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201765.486474:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559765632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201765.486484:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201765.486487:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201765.486491:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559765632:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788201765.486498:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201762/real 1788201765] req@ffff93c0e62b8a80 x1875064559765376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201765.486509:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559765376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201765.486517:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201765.486520:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201765.486523:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559765376:192.168.201.156@tcp:250:kworker.0 00010000:00010000:3.0:1788201766.317995:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201767.149981:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201767.150005:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201767.150106:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201767.150125:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201767.150128:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201767.150147:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201767.342005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201767.661928:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201767.661942:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201767.661947:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201767.661972:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 898 00000100:00080000:2.0:1788201767.661986:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201767.661995:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201767.662050:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559765760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201767.662081:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201767.662086:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201767.662089:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201767.662095:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 898 00000100:00080000:2.0:1788201767.662099:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201767.662104:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201767.662116:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201767.662133:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559765888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201767.662139:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559765760:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788201767.662152:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201767.662157:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201767.662159:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201767.662166:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 898 00000100:00080000:2.0:1788201767.662170:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201767.662175:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201767.662182:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201767.662196:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559766016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201767.662216:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201767.662264:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201767/real 1788201767] req@ffff93c0e62bad80 x1875064559765760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201767.662285:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559765760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201767.662298:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201767.662303:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201767.662308:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559765760:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201767.662333:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559765888:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201767.662353:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201767.662363:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201767/real 1788201767] req@ffff93c0e62b9f80 x1875064559765888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201767.662371:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559765888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201767.662377:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201767.662379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201767.662382:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559765888:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201767.662386:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559766016:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201767.662396:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201767.662402:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201767/real 1788201767] req@ffff93c0e62bb100 x1875064559766016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201767.662415:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559766016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201767.662422:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201767.662424:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201767.662426:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559766016:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788201768.365972:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201769.390025:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201770.414010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201771.438025:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201772.270016:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201772.270034:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201772.270174:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201772.270198:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201772.270203:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201772.270236:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788201772.462010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201772.781912:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201772.781923:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201772.781927:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201772.781937:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 903 00000100:00080000:2.0:1788201772.781948:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201772.781955:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201772.782003:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559766144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201772.782027:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201772.782030:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201772.782031:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201772.782034:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 903 00000100:00080000:2.0:1788201772.782037:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201772.782040:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201772.782051:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201772.782061:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559766272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201772.782069:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559766144:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788201772.782071:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201772.782077:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201772.782080:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201772.782083:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 903 00000100:00080000:2.0:1788201772.782085:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201772.782087:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201772.782091:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201772.782100:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559766400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201772.782198:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559766272:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201772.782231:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559766400:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201773.485964:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201774.510008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788201774.702168:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.156:988 00000400:00000100:1.0:1788201774.702218:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788201774.702236:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788201774.702280:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788201774.702292:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788201774.702325:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201772/real 1788201774] req@ffff93c0e62b9c00 x1875064559766272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201774.702358:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559766272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201774.702373:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201774.702379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201774.702385:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559766272:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201774.702394:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201772/real 1788201774] req@ffff93c0e62bbb80 x1875064559766400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201774.702417:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559766400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201774.702431:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201774.702435:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201774.702439:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559766400:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788201774.702447:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201772/real 1788201774] req@ffff93c0e62ba680 x1875064559766144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201774.702458:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559766144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201774.702465:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201774.702469:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201774.702473:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559766144:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788201775.533927:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201776.558040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788201777.389990:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201777.390004:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201777.390096:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201777.390121:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201777.390124:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201777.390145:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201777.581981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201777.901968:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201777.901980:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201777.901985:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201777.901996:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 908 00000100:00080000:2.0:1788201777.902009:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201777.902018:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201777.902064:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559766528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201777.902096:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201777.902099:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201777.902102:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201777.902106:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 908 00000100:00080000:2.0:1788201777.902111:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201777.902116:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201777.902127:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201777.902141:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559766656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201777.902154:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201777.902162:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1788201777.902162:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559766528:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788201777.902169:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201777.902180:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 908 00000100:00080000:2.0:1788201777.902184:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201777.902189:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201777.902196:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201777.902218:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559766784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201777.902247:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201777.902293:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201777/real 1788201777] req@ffff93c0e62b8380 x1875064559766528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201777.902309:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559766528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201777.902319:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201777.902323:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201777.902327:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559766528:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201777.902351:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559766656:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201777.902368:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201777.902377:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201777/real 1788201777] req@ffff93c0e62b8700 x1875064559766656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201777.902385:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559766656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201777.902391:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201777.902394:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201777.902397:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559766656:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201777.902405:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559766784:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201777.902425:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201777.902437:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201777/real 1788201777] req@ffff93c0e62b8000 x1875064559766784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201777.902451:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559766784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201777.902462:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201777.902464:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201777.902467:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559766784:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788201778.606050:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201779.629962:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201780.654034:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201781.677978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788201782.510007:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201782.510024:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201782.510145:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201782.510182:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201782.510187:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201782.510220:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788201782.702043:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201783.021967:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201783.021982:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201783.021987:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201783.022012:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 913 00000100:00080000:1.0:1788201783.022027:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201783.022036:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788201783.022094:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559766912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201783.022130:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201783.022134:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201783.022136:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201783.022141:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 913 00000100:00080000:1.0:1788201783.022146:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201783.022150:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201783.022163:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201783.022183:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559766912:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788201783.022190:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559767040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201783.022205:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201783.022210:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201783.022212:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201783.022218:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 913 00000100:00080000:1.0:1788201783.022223:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201783.022228:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201783.022234:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788201783.022251:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559767168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201783.022294:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559767040:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201783.022322:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559767168:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201783.726005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788201783.918215:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.156:988 00000400:00000100:0.0:1788201783.918275:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788201783.918296:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788201783.918343:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788201783.918354:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788201783.918388:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201783/real 1788201783] req@ffff93c1c5035c00 x1875064559767040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201783.918418:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559767040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201783.918432:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201783.918437:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201783.918442:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559767040:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201783.918451:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201783/real 1788201783] req@ffff93c1c5036680 x1875064559767168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201783.918463:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559767168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201783.918474:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201783.918479:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201783.918483:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559767168:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788201783.918491:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201783/real 1788201783] req@ffff93c1c5037100 x1875064559766912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201783.918504:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559766912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201783.918518:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201783.918521:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201783.918525:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559766912:192.168.201.156@tcp:250:kworker.0 00010000:00010000:3.0:1788201784.750055:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201785.774019:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201786.797991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201787.630012:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201787.630029:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201787.630141:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201787.630163:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201787.630166:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201787.630183:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201787.821994:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201788.142022:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201788.142032:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201788.142036:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201788.142047:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 919 00000100:00080000:2.0:1788201788.142059:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201788.142066:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201788.142111:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559767296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201788.142139:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201788.142142:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201788.142144:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201788.142147:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 919 00000100:00080000:2.0:1788201788.142149:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201788.142152:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201788.142162:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201788.142189:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559767296:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201788.142201:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559767424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201788.142211:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201788.142214:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201788.142216:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201788.142220:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 919 00000100:00080000:2.0:1788201788.142223:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201788.142227:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201788.142232:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201788.142242:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559767552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201788.142268:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201788.142322:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201788/real 1788201788] req@ffff93c0e62baa00 x1875064559767296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201788.142343:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559767296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201788.142352:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201788.142355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201788.142359:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559767296:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201788.142383:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559767424:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201788.142403:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201788.142412:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201788/real 1788201788] req@ffff93c0e62bb480 x1875064559767424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201788.142420:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559767424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201788.142426:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201788.142429:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201788.142431:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559767424:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201788.142435:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559767552:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201788.142447:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201788.142453:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201788/real 1788201788] req@ffff93c0e62b8000 x1875064559767552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201788.142461:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559767552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201788.142468:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201788.142470:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201788.142472:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559767552:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788201788.846034:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201789.869935:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201790.894025:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201791.918020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201792.750004:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201792.750018:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201792.750107:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201792.750140:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201792.750155:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201792.750192:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788201792.941993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201793.261976:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201793.261985:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201793.261988:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201793.261997:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 924 00000100:00080000:2.0:1788201793.262006:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201793.262014:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201793.262050:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559767680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201793.262076:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201793.262079:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201793.262080:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201793.262083:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 924 00000100:00080000:2.0:1788201793.262086:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201793.262089:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201793.262097:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201793.262105:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559767808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201793.262113:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201793.262115:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201793.262116:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201793.262119:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 924 00000100:00080000:2.0:1788201793.262121:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201793.262123:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201793.262127:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201793.262136:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559767936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201793.262138:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559767680:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788201793.262213:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201793.262249:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201793/real 1788201793] req@ffff93c0e62b8700 x1875064559767680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201793.262263:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559767680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201793.262281:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201793.262287:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201793.262291:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559767680:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201793.262296:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559767808:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201793.262312:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201793.262320:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201793/real 1788201793] req@ffff93c0e62b8380 x1875064559767808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201793.262329:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559767808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201793.262334:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201793.262336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201793.262338:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559767808:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201793.262358:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559767936:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201793.262369:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201793.262375:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201793/real 1788201793] req@ffff93c0e62ba680 x1875064559767936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201793.262382:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559767936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201793.262389:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201793.262391:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201793.262392:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559767936:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201793.966028:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201794.989999:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201796.014008:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201797.037989:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201797.870014:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201797.870033:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201797.870150:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201797.870172:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201797.870175:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201797.870193:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201798.061959:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201798.381944:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201798.381955:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201798.381959:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201798.381971:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 929 00000100:00080000:2.0:1788201798.381984:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201798.381990:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201798.382034:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559768064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201798.382063:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201798.382066:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201798.382068:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201798.382071:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 929 00000100:00080000:2.0:1788201798.382073:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201798.382076:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201798.382087:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201798.382098:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559768192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201798.382110:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201798.382113:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201798.382114:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201798.382117:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 929 00000100:00080000:2.0:1788201798.382119:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201798.382126:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788201798.382128:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559768064:192.168.201.156@tcp:250:kworker.0 00000008:00100000:2.0:1788201798.382131:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201798.382149:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559768320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201798.382212:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201798.382249:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201798/real 1788201798] req@ffff93c0e62bbb80 x1875064559768064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201798.382269:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559768064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201798.382282:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201798.382287:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201798.382292:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559768064:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201798.382318:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559768192:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201798.382338:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201798.382348:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201798/real 1788201798] req@ffff93c0e62b9c00 x1875064559768192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201798.382366:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559768192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201798.382375:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201798.382378:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201798.382380:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559768192:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201798.382385:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559768320:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201798.382399:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201798.382406:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201798/real 1788201798] req@ffff93c0e62bb100 x1875064559768320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201798.382418:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559768320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201798.382441:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201798.382444:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201798.382446:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559768320:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788201799.086030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201800.109984:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201801.133978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201802.157985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201802.989982:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201802.989999:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201802.990118:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201802.990150:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201802.990156:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201802.990180:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788201803.182010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201803.501943:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201803.501958:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201803.501963:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201803.501976:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 934 00000100:00080000:2.0:1788201803.501988:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201803.501996:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201803.502039:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559768448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201803.502066:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201803.502069:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201803.502071:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201803.502077:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 934 00000100:00080000:2.0:1788201803.502080:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201803.502085:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201803.502096:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201803.502111:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559768576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201803.502124:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201803.502128:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201803.502131:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201803.502135:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 934 00000100:00080000:2.0:1788201803.502138:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201803.502142:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201803.502147:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201803.502163:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559768704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201803.502164:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559768448:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201803.502281:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559768576:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201803.502310:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559768704:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201804.205993:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201805.230004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788201805.422184:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788201805.422250:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788201805.422271:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788201805.422335:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788201805.422348:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788201805.422412:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201803/real 1788201805] req@ffff93c0e62b8a80 x1875064559768704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201805.422441:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559768704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201805.422456:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201805.422462:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201805.422467:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559768704:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788201805.422477:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201803/real 1788201805] req@ffff93c0e62b9f80 x1875064559768448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201805.422498:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559768448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201805.422505:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201805.422509:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201805.422512:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559768448:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201805.422519:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201803/real 1788201805] req@ffff93c0e62bad80 x1875064559768576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201805.422530:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559768576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201805.422537:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201805.422540:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201805.422543:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559768576:192.168.201.156@tcp:38:kworker.0 00010000:00010000:0.0:1788201806.253941:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201807.277978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788201808.110049:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201808.110064:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201808.110152:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201808.110173:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201808.110176:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201808.110194:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201808.301949:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201808.622007:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201808.622018:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201808.622022:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201808.622034:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 939 00000100:00080000:3.0:1788201808.622045:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201808.622053:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788201808.622099:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c53e0000 x1875064559768832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201808.622130:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201808.622133:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201808.622135:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201808.622138:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 939 00000100:00080000:3.0:1788201808.622140:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201808.622143:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201808.622152:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201808.622162:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c53e2a00 x1875064559768960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201808.622169:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201808.622172:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201808.622175:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201808.622178:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 939 00000100:00080000:3.0:1788201808.622180:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201808.622183:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201808.622187:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201808.622194:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c53e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559768832:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201808.622235:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ca80 x1875064559769088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201808.622285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201808.622332:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201808/real 1788201808] req@ffff93c1c53e0000 x1875064559768832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201808.622352:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c53e0000 x1875064559768832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201808.622361:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201808.622364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201808.622368:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c53e0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559768832:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201808.622374:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c53e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559768960:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201808.622391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201808.622399:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201808/real 1788201808] req@ffff93c1c53e2a00 x1875064559768960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201808.622407:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c53e2a00 x1875064559768960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201808.622417:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201808.622420:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201808.622422:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c53e2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559768960:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201808.622443:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559769088:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201808.622456:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201808.622462:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201808/real 1788201808] req@ffff93c1c4a4ca80 x1875064559769088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201808.622469:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ca80 x1875064559769088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201808.622476:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201808.622478:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201808.622480:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559769088:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788201809.326050:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201810.349905:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201811.374068:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201812.397959:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788201813.229972:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201813.229989:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201813.230084:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201813.230113:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201813.230117:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201813.230147:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788201813.422006:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201813.741955:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201813.741966:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201813.741970:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201813.741980:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 944 00000100:00080000:1.0:1788201813.741992:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201813.741999:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788201813.742046:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559769216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201813.742089:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201813.742092:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201813.742094:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201813.742098:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 944 00000100:00080000:1.0:1788201813.742102:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201813.742106:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201813.742116:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201813.742129:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559769344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201813.742143:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559769216:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788201813.742146:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201813.742152:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201813.742174:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201813.742180:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 944 00000100:00080000:1.0:1788201813.742183:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201813.742186:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201813.742190:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788201813.742203:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559769472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201813.742253:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201813.742292:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201813/real 1788201813] req@ffff93c1c5034a80 x1875064559769216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201813.742306:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559769216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201813.742314:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201813.742317:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201813.742320:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559769216:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201813.742343:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559769344:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201813.742373:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201813.742383:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201813/real 1788201813] req@ffff93c1c5034e00 x1875064559769344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201813.742391:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559769344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201813.742396:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201813.742399:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201813.742401:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559769344:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201813.742405:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559769472:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201813.742417:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201813.742423:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201813/real 1788201813] req@ffff93c1c5037100 x1875064559769472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201813.742430:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559769472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201813.742437:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201813.742439:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201813.742441:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559769472:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201814.445982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201815.470036:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201816.493945:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201817.518030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788201818.349994:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201818.350014:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201818.350178:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201818.350213:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201818.350217:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201818.350242:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201818.541928:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201818.861976:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201818.861987:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201818.861991:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201818.862002:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 949 00000100:00080000:2.0:1788201818.862014:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201818.862022:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201818.862064:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559769600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201818.862104:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201818.862107:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201818.862109:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201818.862112:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 949 00000100:00080000:2.0:1788201818.862115:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201818.862119:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201818.862129:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201818.862142:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559769728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201818.862150:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201818.862154:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201818.862156:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788201818.862158:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559769600:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788201818.862163:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 949 00000100:00080000:2.0:1788201818.862168:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201818.862172:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201818.862176:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201818.862184:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559769856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201818.862241:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201818.862289:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201818/real 1788201818] req@ffff93c0e62b9500 x1875064559769600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201818.862309:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559769600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201818.862322:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201818.862327:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201818.862332:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559769600:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201818.862359:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559769728:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201818.862383:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201818.862396:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201818/real 1788201818] req@ffff93c0e62bb800 x1875064559769728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201818.862410:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559769728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201818.862420:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201818.862424:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201818.862428:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559769728:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201818.862435:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559769856:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201818.862454:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201818.862465:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201818/real 1788201818] req@ffff93c0e62bad80 x1875064559769856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201818.862478:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559769856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201818.862490:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201818.862493:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201818.862497:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559769856:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788201819.566113:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201820.589963:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201821.614050:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201822.637952:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788201823.470033:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201823.470050:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201823.470163:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201823.470188:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201823.470191:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201823.470209:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788201823.662068:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201823.981949:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201823.981959:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201823.981963:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201823.981974:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 954 00000100:00080000:3.0:1788201823.982001:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201823.982010:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788201823.982054:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d500 x1875064559769984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201823.982081:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201823.982084:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201823.982085:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201823.982090:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 954 00000100:00080000:3.0:1788201823.982093:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201823.982096:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201823.982106:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201823.982116:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559769984:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201823.982118:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559770112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201823.982132:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201823.982134:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201823.982135:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201823.982139:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 954 00000100:00080000:3.0:1788201823.982140:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201823.982143:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201823.982148:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788201823.982157:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559770240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201823.982198:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201823.982239:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201823/real 1788201823] req@ffff93c1c4a4d500 x1875064559769984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201823.982256:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d500 x1875064559769984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201823.982265:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201823.982269:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201823.982272:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559769984:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201823.982304:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559770112:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201823.982327:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201823.982341:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201823/real 1788201823] req@ffff93c1c4a4f800 x1875064559770112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201823.982355:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559770112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201823.982363:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201823.982365:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201823.982368:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559770112:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201823.982372:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559770240:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201823.982389:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201823.982397:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201823/real 1788201823] req@ffff93c1c4a4d180 x1875064559770240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201823.982404:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559770240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201823.982412:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201823.982414:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201823.982415:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559770240:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201824.685943:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201825.709959:0:13364:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 4/4 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1788201825.709982:0:10052:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 4 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1788201825.709993:0:10052:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff93c1d85e6000 lock: ffff93c0e61ea600/0x390884812bd99b66 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xa81c4c9a2f3cb745 expref: -99 pid: 17463 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1788201825.710000:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788201825.710051:0:10052:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x10:0x0]: losing layout lock 00010000:00010000:3.0:1788201825.710084:0:10052:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff93c1d85e6000 lock: ffff93c1c8592a00/0x390884812bd99b4a lrc: 2/0,0 mode: CW/CW res: [0x200000402:0x10:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xa81c4c9a2f3cb67a expref: -99 pid: 17463 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788201825.710094:0:10052:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff93c1d85e6000 lock: ffff93c1c8593e00/0x390884812bd99b51 lrc: 2/0,0 mode: CR/CR res: [0x200000402:0x10:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xa81c4c9a2f3cb68f expref: -99 pid: 17463 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788201825.710104:0:10052:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x10:0x0]: losing layout lock 00010000:00010000:3.0:1788201825.710111:0:10052:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff93c1d85e6000 lock: ffff93c1c0030a00/0x390884812bd99944 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x1:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xa81c4c9a2f3cadac expref: -99 pid: 11440 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788201825.710144:0:10052:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff93c1d85e6000 lock: ffff93c0e61ea600/0x390884812bd99b66 lrc: 1/0,0 mode: --/PR res: [0x200000402:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xa81c4c9a2f3cb745 expref: -99 pid: 17463 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788201825.710150:0:10052:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff93c1d85e6000 lock: ffff93c1c8592a00/0x390884812bd99b4a lrc: 1/0,0 mode: --/CW res: [0x200000402:0x10:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xa81c4c9a2f3cb67a expref: -99 pid: 17463 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788201825.710154:0:10052:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff93c1d85e6000 lock: ffff93c1c8593e00/0x390884812bd99b51 lrc: 1/0,0 mode: --/CR res: [0x200000402:0x10:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xa81c4c9a2f3cb68f expref: -99 pid: 17463 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788201825.710158:0:10052:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff93c1d85e6000 lock: ffff93c1c0030a00/0x390884812bd99944 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x1:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xa81c4c9a2f3cadac expref: -99 pid: 11440 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788201825.710162:0:10052:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 4 locks packed 00010000:00010000:3.0:1788201825.710192:0:10052:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff93c1d85e6000 lock: ffff93c0e61ea600/0x390884812bd99b66 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xa81c4c9a2f3cb745 expref: -99 pid: 17463 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788201825.710209:0:10052:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff93c1d85e6000 lock: ffff93c1c8592a00/0x390884812bd99b4a lrc: 0/0,0 mode: --/CW res: [0x200000402:0x10:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xa81c4c9a2f3cb67a expref: -99 pid: 17463 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788201825.710220:0:10052:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff93c1d85e6000 lock: ffff93c1c8593e00/0x390884812bd99b51 lrc: 0/0,0 mode: --/CR res: [0x200000402:0x10:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xa81c4c9a2f3cb68f expref: -99 pid: 17463 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000100:00080000:2.0:1788201825.710238:0:2343:0:(client.c:1832:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff93c1c4a4ce00 x1875064559770368/t0(0) o103->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:17/18 lens 344/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1788201825.710238:0:10052:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff93c1d85e6000 lock: ffff93c1c0030a00/0x390884812bd99944 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x1:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xa81c4c9a2f3cadac expref: -99 pid: 11440 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788201825.710260:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201825.710263:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201825.710265:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201826.733961:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201827.758054:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788201828.590018:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201828.590052:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201828.590201:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201828.590394:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201828.590400:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201828.590453:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201828.781969:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201829.101935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201829.101946:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201829.101951:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201829.101962:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 959 00000100:00080000:3.0:1788201829.101974:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201829.101981:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788201829.102028:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559770496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201829.102062:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201829.102067:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201829.102069:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201829.102073:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 959 00000100:00080000:3.0:1788201829.102077:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201829.102080:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201829.102092:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201829.102107:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559770624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201829.102114:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559770496:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788201829.102127:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201829.102131:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201829.102133:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201829.102138:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 959 00000100:00080000:3.0:1788201829.102140:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201829.102144:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201829.102148:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788201829.102158:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559770752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201829.102199:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201829.102243:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201829/real 1788201829] req@ffff93c1c4a4d880 x1875064559770496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201829.102258:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559770496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201829.102266:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201829.102270:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201829.102273:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559770496:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201829.102294:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559770624:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201829.102312:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201829.102321:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201829/real 1788201829] req@ffff93c1c4a4e300 x1875064559770624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201829.102334:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559770624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201829.102339:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201829.102342:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201829.102344:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559770624:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201829.102348:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559770752:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201829.102359:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201829.102365:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201829/real 1788201829] req@ffff93c1c4a4f480 x1875064559770752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201829.102372:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559770752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201829.102379:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201829.102381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201829.102382:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559770752:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788201829.806033:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201830.830028:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201831.854049:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201832.877915:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788201833.709973:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201833.709987:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201833.710068:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201833.710088:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201833.710091:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201833.710108:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788201833.901994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201834.221957:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201834.221974:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201834.221982:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201834.221994:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 965 00000100:00080000:1.0:1788201834.222006:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201834.222014:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788201834.222057:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559770880/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201834.222080:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201834.222083:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201834.222084:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201834.222087:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 965 00000100:00080000:1.0:1788201834.222089:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201834.222092:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201834.222100:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201834.222110:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559771008/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201834.222117:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201834.222120:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201834.222121:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201834.222125:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 965 00000100:00080000:1.0:1788201834.222129:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201834.222134:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201834.222139:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788201834.222153:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559771136/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201834.222173:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559770880:192.168.201.156@tcp:250:kworker.0 00000800:00000100:2.0:1788201834.222260:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201834.222298:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201834/real 1788201834] req@ffff93c1c5036680 x1875064559770880/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201834.222313:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559770880/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201834.222327:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201834.222330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201834.222333:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559770880:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201834.222339:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559771008:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201834.222356:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201834.222364:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201834/real 1788201834] req@ffff93c1c5035c00 x1875064559771008/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201834.222372:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559771008/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201834.222377:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201834.222380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201834.222382:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559771008:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201834.222408:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559771136:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201834.222431:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201834.222443:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201834/real 1788201834] req@ffff93c1c5037800 x1875064559771136/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201834.222452:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559771136/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201834.222459:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201834.222462:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201834.222464:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559771136:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201834.926050:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201835.949995:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201836.973971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201837.997977:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788201838.829974:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201838.829989:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201838.830068:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201838.830098:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201838.830102:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201838.830121:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201839.021903:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201839.341946:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201839.341956:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201839.341960:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201839.341969:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 970 00000100:00080000:1.0:1788201839.341985:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201839.341992:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788201839.342028:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559771264/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201839.342053:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201839.342057:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201839.342059:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201839.342063:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 970 00000100:00080000:1.0:1788201839.342067:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201839.342072:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201839.342082:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201839.342097:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559771392/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201839.342107:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559771264:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788201839.342115:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201839.342118:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201839.342120:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201839.342124:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 970 00000100:00080000:1.0:1788201839.342127:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201839.342130:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201839.342135:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788201839.342144:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559771520/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201839.342206:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559771392:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201839.342233:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559771520:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788201840.046016:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201841.070024:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201842.093989:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788201842.414153:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788201842.414201:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788201842.414220:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788201842.414270:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788201842.414281:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:2.0:1788201842.414339:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201839/real 1788201842] req@ffff93c1c5035c00 x1875064559771392/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201842.414376:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559771392/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201842.414389:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201842.414395:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201842.414400:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559771392:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201842.414409:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201839/real 1788201842] req@ffff93c1c5036680 x1875064559771520/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201842.414424:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559771520/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201842.414434:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201842.414438:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201842.414442:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559771520:192.168.201.156@tcp:8:kworker.0 00000100:00100000:2.0:1788201842.414451:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201839/real 1788201842] req@ffff93c1c5037800 x1875064559771264/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201842.414464:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559771264/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201842.414472:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201842.414476:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201842.414479:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559771264:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788201843.118019:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201843.950016:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201843.950031:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201843.950115:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201843.950132:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201843.950134:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201843.950152:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788201844.142035:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201844.461955:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201844.461964:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201844.461968:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201844.461978:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 975 00000100:00080000:2.0:1788201844.461991:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201844.462000:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201844.462042:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559771648/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201844.462075:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201844.462079:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201844.462081:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201844.462085:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 975 00000100:00080000:2.0:1788201844.462089:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201844.462093:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201844.462105:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201844.462118:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559771776/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201844.462129:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788201844.462129:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559771648:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788201844.462135:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201844.462140:0:38:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201844.462144:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 975 00000100:00080000:2.0:1788201844.462146:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201844.462149:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201844.462154:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201844.462163:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559771904/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201844.462210:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201844.462260:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201844/real 1788201844] req@ffff93c0e62b9f80 x1875064559771648/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201844.462282:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559771648/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201844.462294:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201844.462299:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201844.462304:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559771648:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201844.462331:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559771776:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201844.462355:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201844.462369:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201844/real 1788201844] req@ffff93c0e62b8a80 x1875064559771776/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201844.462398:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559771776/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201844.462407:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201844.462412:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201844.462416:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559771776:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201844.462423:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559771904:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201844.462443:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201844.462456:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201844/real 1788201844] req@ffff93c0e62bb100 x1875064559771904/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201844.462477:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559771904/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201844.462489:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201844.462493:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201844.462497:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559771904:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201845.166004:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201846.190034:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201847.214029:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201848.238033:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788201849.070066:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201849.070083:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201849.070420:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201849.070461:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201849.070468:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201849.070506:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201849.261991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201849.581983:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201849.581992:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201849.581996:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201849.582007:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 980 00000100:00080000:3.0:1788201849.582019:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201849.582025:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788201849.582066:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559772032/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201849.582091:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201849.582094:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201849.582096:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201849.582100:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 980 00000100:00080000:3.0:1788201849.582103:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201849.582106:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201849.582116:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201849.582128:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559772032:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201849.582128:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559772160/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201849.582143:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201849.582145:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201849.582147:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201849.582150:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 980 00000100:00080000:3.0:1788201849.582152:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201849.582154:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201849.582158:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788201849.582168:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559772288/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201849.582209:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201849.582249:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201849/real 1788201849] req@ffff93c1c4a4c700 x1875064559772032/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201849.582264:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559772032/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201849.582273:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201849.582277:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201849.582280:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559772032:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201849.582302:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559772160:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201849.582317:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201849.582325:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201849/real 1788201849] req@ffff93c1c4a4c380 x1875064559772160/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201849.582333:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559772160/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201849.582339:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201849.582340:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201849.582342:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559772160:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201849.582346:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559772288:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201849.582358:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201849.582364:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201849/real 1788201849] req@ffff93c1c4a4ea00 x1875064559772288/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201849.582371:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559772288/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201849.582379:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201849.582380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201849.582382:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559772288:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788201850.286055:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201851.309971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201852.334078:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201853.357960:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788201854.190013:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201854.190033:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201854.190125:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201854.190152:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201854.190155:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201854.190175:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788201854.382020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201854.701942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201854.701953:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201854.701958:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201854.701969:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 985 00000100:00080000:2.0:1788201854.701982:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201854.701991:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201854.702034:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559772416/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201854.702065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201854.702070:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201854.702072:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201854.702077:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 985 00000100:00080000:2.0:1788201854.702082:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201854.702087:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201854.702097:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201854.702106:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559772416:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201854.702113:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559772544/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201854.702125:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201854.702128:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201854.702129:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201854.702133:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 985 00000100:00080000:2.0:1788201854.702135:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201854.702138:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201854.702143:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201854.702152:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559772672/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201854.702179:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201854.702221:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201854/real 1788201854] req@ffff93c0e62b9c00 x1875064559772416/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201854.702236:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559772416/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201854.702245:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201854.702248:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201854.702251:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559772416:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201854.702271:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559772544:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201854.702288:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201854.702297:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201854/real 1788201854] req@ffff93c0e62bbb80 x1875064559772544/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201854.702305:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559772544/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201854.702311:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201854.702313:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201854.702315:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559772544:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201854.702319:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559772672:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201854.702330:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201854.702336:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201854/real 1788201854] req@ffff93c0e62ba680 x1875064559772672/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201854.702347:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559772672/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201854.702354:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201854.702356:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201854.702358:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559772672:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201855.405963:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201856.430015:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201857.453960:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201858.477989:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788201859.310018:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201859.310035:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201859.310188:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201859.310221:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201859.310226:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201859.310260:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201859.501945:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788201859.821950:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201859.821960:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788201859.821964:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201859.821973:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 990 00000100:00080000:0.0:1788201859.821983:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201859.821989:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788201859.822024:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c8485880 x1875064559772800/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201859.822048:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201859.822051:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788201859.822052:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201859.822055:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 990 00000100:00080000:0.0:1788201859.822057:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201859.822060:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788201859.822068:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201859.822088:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c8485880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559772800:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788201859.822103:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559772928/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201859.822113:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201859.822116:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788201859.822118:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201859.822121:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 990 00000100:00080000:0.0:1788201859.822124:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201859.822127:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788201859.822131:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788201859.822138:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559773056/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201859.822161:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201859.822206:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201859/real 1788201859] req@ffff93c1c8485880 x1875064559772800/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201859.822228:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c8485880 x1875064559772800/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201859.822240:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201859.822244:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201859.822248:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c8485880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559772800:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201859.822290:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559772928:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201859.822318:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201859.822332:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201859/real 1788201859] req@ffff93c0e9ce7480 x1875064559772928/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201859.822341:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559772928/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201859.822348:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201859.822350:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201859.822353:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559772928:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201859.822374:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559773056:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201859.822391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201859.822400:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201859/real 1788201859] req@ffff93c0e9ce4700 x1875064559773056/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201859.822408:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559773056/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201859.822415:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201859.822417:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201859.822419:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559773056:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788201860.525997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201861.549931:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201862.574030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201863.597971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788201864.429989:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201864.430003:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201864.430089:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201864.430109:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201864.430112:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201864.430130:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788201864.622049:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788201864.941988:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201864.941999:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788201864.942003:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201864.942013:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 995 00000100:00080000:0.0:1788201864.942023:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201864.942030:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788201864.942068:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559773184/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201864.942090:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201864.942093:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788201864.942094:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201864.942098:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 995 00000100:00080000:0.0:1788201864.942101:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201864.942104:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788201864.942113:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201864.942121:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559773184:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788201864.942124:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559773312/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201864.942137:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201864.942140:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788201864.942141:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201864.942143:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 995 00000100:00080000:0.0:1788201864.942147:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201864.942151:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788201864.942155:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788201864.942164:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559773440/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201864.942192:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201864.942227:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201864/real 1788201864] req@ffff93c0e9ce4380 x1875064559773184/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201864.942242:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559773184/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201864.942255:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201864.942291:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201864.942295:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559773184:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201864.942317:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559773312:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201864.942336:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201864.942346:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201864/real 1788201864] req@ffff93c0e9ce6a00 x1875064559773312/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201864.942356:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559773312/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201864.942362:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201864.942364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201864.942367:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559773312:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201864.942371:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559773440:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201864.942383:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201864.942390:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201864/real 1788201864] req@ffff93c0e9ce4000 x1875064559773440/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201864.942397:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559773440/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201864.942405:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201864.942407:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201864.942409:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559773440:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201865.646033:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201866.670064:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201867.694007:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201868.718068:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201869.550065:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201869.550083:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201869.550214:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201869.550237:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201869.550240:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201869.550261:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201869.742065:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201870.061926:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201870.061941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201870.061946:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201870.061960:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1000 00000100:00080000:2.0:1788201870.061973:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201870.061980:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201870.062025:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559773568/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201870.062050:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201870.062055:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201870.062057:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201870.062062:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1000 00000100:00080000:2.0:1788201870.062078:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201870.062084:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201870.062098:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201870.062102:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559773568:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201870.062117:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559773696/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201870.062127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201870.062130:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201870.062132:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201870.062147:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1000 00000100:00080000:2.0:1788201870.062160:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201870.062165:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201870.062171:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201870.062199:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559773824/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201870.062211:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201870.062252:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201870/real 1788201870] req@ffff93c0e62b8380 x1875064559773568/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201870.062268:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559773568/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201870.062278:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201870.062288:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201870.062291:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559773568:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201870.062314:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559773696:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201870.062329:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201870.062338:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201870/real 1788201870] req@ffff93c0e62b8700 x1875064559773696/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201870.062346:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559773696/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201870.062352:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201870.062354:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201870.062357:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559773696:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201870.062360:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559773824:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201870.062374:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201870.062380:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201870/real 1788201870] req@ffff93c0e62b8000 x1875064559773824/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201870.062388:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559773824/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201870.062396:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201870.062397:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201870.062399:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559773824:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788201870.765996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201871.789948:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788201872.814082:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201873.838015:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788201874.670036:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201874.670054:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201874.670223:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201874.670255:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201874.670259:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201874.670290:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201874.861986:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201875.181967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201875.181982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201875.181987:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201875.182000:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1006 00000100:00080000:2.0:1788201875.182016:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201875.182026:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201875.182079:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559773952/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201875.182142:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201875.182146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201875.182148:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201875.182153:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1006 00000100:00080000:2.0:1788201875.182157:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201875.182161:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201875.182173:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201875.182191:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559774080/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201875.182202:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201875.182207:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201875.182209:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201875.182215:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1006 00000100:00080000:2.0:1788201875.182221:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201875.182226:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201875.182231:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788201875.182238:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559773952:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201875.182249:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559774208/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201875.182327:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201875.182373:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201875/real 1788201875] req@ffff93c0e62bb480 x1875064559773952/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201875.182389:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559773952/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201875.182397:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201875.182400:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201875.182404:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559773952:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201875.182410:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559774080:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201875.182440:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201875.182453:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201875/real 1788201875] req@ffff93c0e62baa00 x1875064559774080/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201875.182467:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559774080/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201875.182475:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201875.182479:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201875.182483:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559774080:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201875.182507:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559774208:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201875.182523:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201875.182531:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201875/real 1788201875] req@ffff93c0e62b8000 x1875064559774208/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201875.182539:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559774208/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201875.182546:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201875.182548:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201875.182550:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559774208:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201875.886046:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201876.909991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201877.933999:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201878.957938:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201879.789992:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201879.790031:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201879.790148:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201879.790179:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201879.790185:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201879.790219:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201879.981955:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201880.301957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201880.301967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201880.301972:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201880.301981:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1011 00000100:00080000:2.0:1788201880.301991:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201880.301998:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201880.302061:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559774336/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201880.302085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201880.302088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201880.302091:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201880.302096:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1011 00000100:00080000:2.0:1788201880.302100:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201880.302105:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201880.302116:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201880.302132:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559774336:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201880.302134:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559774464/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201880.302157:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201880.302160:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201880.302163:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201880.302168:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1011 00000100:00080000:2.0:1788201880.302188:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788201880.302197:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00080000:2.0:1788201880.302203:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201880.302211:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201880.302225:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559774592/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201880.302238:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201880/real 1788201880] req@ffff93c0e62b8700 x1875064559774336/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201880.302254:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559774336/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201880.302267:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201880.302271:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201880.302274:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559774336:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201880.302293:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559774464:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201880.302316:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201880.302330:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201880/real 1788201880] req@ffff93c0e62b8380 x1875064559774464/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201880.302344:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559774464/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201880.302350:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201880.302352:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201880.302355:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559774464:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201880.302359:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559774592:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201880.302388:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201880.302396:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201880/real 1788201880] req@ffff93c0e62ba680 x1875064559774592/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201880.302403:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559774592/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201880.302410:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201880.302412:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201880.302414:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559774592:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201881.005970:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201882.029900:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201883.053990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201884.077957:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788201884.910044:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201884.910059:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201884.910148:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201884.910169:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201884.910172:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201884.910191:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201885.102000:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788201885.421989:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201885.422000:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788201885.422004:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201885.422015:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1016 00000100:00080000:0.0:1788201885.422026:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201885.422033:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788201885.422075:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559774720/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201885.422105:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201885.422108:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788201885.422110:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201885.422117:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1016 00000100:00080000:0.0:1788201885.422119:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201885.422122:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788201885.422132:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788201885.422144:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559774848/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201885.422153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201885.422157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788201885.422160:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788201885.422161:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559774720:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788201885.422167:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1016 00000100:00080000:0.0:1788201885.422170:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201885.422173:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788201885.422177:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788201885.422186:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559774976/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201885.422231:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201885.422274:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201885/real 1788201885] req@ffff93c0e9ce6680 x1875064559774720/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201885.422289:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559774720/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201885.422298:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201885.422301:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201885.422304:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559774720:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201885.422326:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559774848:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201885.422340:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201885.422348:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201885/real 1788201885] req@ffff93c0e9ce7b80 x1875064559774848/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201885.422356:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559774848/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201885.422362:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201885.422364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201885.422366:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559774848:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201885.422370:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559774976:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201885.422381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201885.422387:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201885/real 1788201885] req@ffff93c0e9ce5c00 x1875064559774976/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201885.422394:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559774976/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201885.422401:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201885.422403:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201885.422405:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559774976:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201886.125974:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201887.150007:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201888.174023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201889.198046:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788201890.030032:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201890.030046:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201890.030159:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201890.030196:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201890.030203:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201890.030249:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201890.221971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201890.541954:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201890.541967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201890.541971:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201890.541983:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1021 00000100:00080000:2.0:1788201890.541996:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201890.542004:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201890.542049:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559775104/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201890.542087:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201890.542091:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201890.542093:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201890.542098:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1021 00000100:00080000:2.0:1788201890.542101:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201890.542105:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201890.542117:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201890.542130:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559775232/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201890.542135:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559775104:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788201890.542150:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201890.542154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201890.542156:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201890.542161:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1021 00000100:00080000:2.0:1788201890.542164:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201890.542168:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201890.542173:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201890.542184:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559775360/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201890.542215:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201890.542260:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201890/real 1788201890] req@ffff93c0e62bbb80 x1875064559775104/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201890.542275:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559775104/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201890.542284:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201890.542287:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201890.542290:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559775104:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201890.542315:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559775232:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201890.542340:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201890.542353:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201890/real 1788201890] req@ffff93c0e62b9c00 x1875064559775232/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201890.542366:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559775232/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201890.542377:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201890.542379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201890.542382:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559775232:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201890.542386:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559775360:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201890.542403:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201890.542410:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201890/real 1788201890] req@ffff93c0e62bb100 x1875064559775360/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201890.542418:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559775360/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201890.542432:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201890.542434:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201890.542436:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559775360:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201891.245995:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201892.269961:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201893.293976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201894.317949:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788201895.150023:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201895.150041:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201895.150144:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201895.150176:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201895.150181:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201895.150215:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201895.341987:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788201895.661941:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201895.661950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788201895.661954:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201895.661962:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1026 00000100:00080000:0.0:1788201895.661973:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201895.661979:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788201895.662015:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559775488/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201895.662037:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201895.662040:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788201895.662042:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788201895.662044:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1026 00000100:00080000:0.0:1788201895.662047:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201895.662049:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788201895.662058:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788201895.662067:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559775616/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788201895.662075:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788201895.662079:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788201895.662080:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788201895.662080:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559775488:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788201895.662086:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1026 00000100:00080000:0.0:1788201895.662091:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788201895.662095:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788201895.662098:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788201895.662105:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559775744/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201895.662152:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201895.662187:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201895/real 1788201895] req@ffff93c0e9ce6d80 x1875064559775488/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201895.662201:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559775488/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201895.662209:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201895.662212:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201895.662215:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559775488:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201895.662237:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559775616:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201895.662252:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201895.662261:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201895/real 1788201895] req@ffff93c0e9ce7100 x1875064559775616/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201895.662270:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559775616/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201895.662275:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201895.662278:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201895.662280:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559775616:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201895.662284:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559775744:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201895.662298:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201895.662309:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201895/real 1788201895] req@ffff93c0e9ce5f80 x1875064559775744/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201895.662322:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559775744/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201895.662333:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201895.662337:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201895.662340:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559775744:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201896.365970:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201897.390001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201898.413980:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201899.437954:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788201900.270004:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201900.270022:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201900.270138:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788201900.270162:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788201900.270165:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788201900.270184:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201900.461949:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201900.781958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201900.781968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201900.781972:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201900.781980:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1031 00000100:00080000:1.0:1788201900.781990:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201900.781998:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788201900.782030:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559775872/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201900.782051:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201900.782054:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201900.782056:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201900.782059:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1031 00000100:00080000:1.0:1788201900.782061:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201900.782063:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201900.782071:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201900.782081:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559776000/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201900.782089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201900.782093:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201900.782095:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788201900.782095:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559775872:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788201900.782101:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1031 00000100:00080000:1.0:1788201900.782106:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201900.782109:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201900.782112:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788201900.782121:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559776128/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201900.782170:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201900.782211:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201900/real 1788201900] req@ffff93c1c5037100 x1875064559775872/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201900.782232:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559775872/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201900.782245:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201900.782249:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201900.782254:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559775872:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201900.782277:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559776000:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201900.782301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201900.782314:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201900/real 1788201900] req@ffff93c1c5034e00 x1875064559776000/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201900.782329:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559776000/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201900.782339:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201900.782342:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201900.782346:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559776000:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201900.782364:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559776128:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201900.782391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201900.782403:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201900/real 1788201900] req@ffff93c1c5034a80 x1875064559776128/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201900.782416:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559776128/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201900.782435:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201900.782439:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201900.782443:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559776128:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201901.485987:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201902.509965:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201903.534019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201904.557971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201905.390048:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201905.390070:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201905.390195:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201905.390224:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201905.390230:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201905.390264:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201905.582019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201905.901995:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201905.902006:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201905.902011:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201905.902023:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1036 00000100:00080000:2.0:1788201905.902034:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201905.902041:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201905.902085:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559776256/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201905.902110:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201905.902113:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201905.902114:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201905.902118:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1036 00000100:00080000:2.0:1788201905.902120:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201905.902122:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201905.902132:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201905.902141:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559776384/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201905.902149:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201905.902153:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201905.902155:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201905.902158:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1036 00000100:00080000:2.0:1788201905.902160:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201905.902162:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201905.902166:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201905.902176:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559776512/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201905.902194:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559776256:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201905.902325:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559776384:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201905.902381:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559776512:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201906.605987:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201907.629981:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201908.653992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788201908.974095:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788201908.974146:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788201908.974163:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788201908.974207:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788201908.974218:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788201908.974268:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201905/real 1788201908] req@ffff93c0e62bad80 x1875064559776512/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201908.974308:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559776512/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201908.974324:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201908.974330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201908.974335:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559776512:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788201908.974345:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201905/real 1788201908] req@ffff93c0e62b8a80 x1875064559776256/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201908.974359:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559776256/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201908.974368:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201908.974372:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201908.974376:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559776256:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201908.974383:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201905/real 1788201908] req@ffff93c0e62b9f80 x1875064559776384/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201908.974396:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559776384/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201908.974403:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201908.974407:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201908.974410:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559776384:192.168.201.156@tcp:38:kworker.0 00010000:00010000:0.0:1788201909.678001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788201910.510007:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201910.510024:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201910.510137:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201910.510171:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201910.510177:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201910.510209:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201910.702000:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201911.021946:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201911.021955:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201911.021958:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201911.021967:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1041 00000100:00080000:1.0:1788201911.021977:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201911.021986:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788201911.022019:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559776640/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201911.022046:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201911.022050:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201911.022052:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201911.022056:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1041 00000100:00080000:1.0:1788201911.022060:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201911.022064:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201911.022075:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201911.022090:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559776768/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201911.022101:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201911.022105:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201911.022107:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201911.022114:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1041 00000100:00080000:1.0:1788201911.022118:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201911.022122:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201911.022129:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788201911.022140:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559776640:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788201911.022144:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559776896/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201911.022219:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201911.022269:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201911/real 1788201911] req@ffff93c1c5034a80 x1875064559776640/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201911.022290:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559776640/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201911.022301:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201911.022306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201911.022311:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559776640:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201911.022320:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559776768:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201911.022351:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201911.022366:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201911/real 1788201911] req@ffff93c1c5034e00 x1875064559776768/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201911.022378:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559776768/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201911.022387:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201911.022390:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201911.022394:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559776768:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201911.022420:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559776896:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201911.022442:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201911.022453:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201911/real 1788201911] req@ffff93c1c5037100 x1875064559776896/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201911.022483:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559776896/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201911.022507:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201911.022511:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201911.022514:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559776896:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201911.726005:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201912.750026:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201913.773961:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201914.797983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201915.630002:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201915.630020:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201915.630138:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201915.630156:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201915.630159:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201915.630186:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201915.821974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201916.141990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201916.142000:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201916.142004:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201916.142015:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1047 00000100:00080000:2.0:1788201916.142027:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201916.142035:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201916.142078:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559777024/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201916.142106:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201916.142115:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201916.142116:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201916.142120:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1047 00000100:00080000:2.0:1788201916.142123:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201916.142126:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201916.142136:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201916.142149:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559777152/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201916.142156:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201916.142159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201916.142160:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201916.142165:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1047 00000100:00080000:2.0:1788201916.142167:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201916.142171:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201916.142177:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788201916.142186:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559777024:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201916.142196:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559777280/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201916.142279:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201916.142350:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201916/real 1788201916] req@ffff93c0e62bb800 x1875064559777024/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201916.142374:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559777024/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201916.142417:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201916.142423:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201916.142428:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559777024:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201916.142438:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559777152:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201916.142465:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201916.142480:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201916/real 1788201916] req@ffff93c0e62b9500 x1875064559777152/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201916.142493:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559777152/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201916.142501:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201916.142505:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201916.142508:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559777152:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201916.142534:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559777280:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201916.142554:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201916.142565:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201916/real 1788201916] req@ffff93c0e62b9f80 x1875064559777280/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201916.142577:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559777280/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201916.142586:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201916.142590:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201916.142593:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559777280:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201916.846064:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201917.869977:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201918.894045:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201919.918000:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201920.750012:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201920.750029:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201920.750148:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201920.750177:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201920.750183:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201920.750224:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201920.941972:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201921.261960:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201921.261971:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201921.261975:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201921.261985:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1052 00000100:00080000:2.0:1788201921.261998:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201921.262005:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201921.262051:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559777408/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201921.262077:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201921.262080:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201921.262081:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201921.262085:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1052 00000100:00080000:2.0:1788201921.262087:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201921.262091:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201921.262103:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201921.262118:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559777536/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201921.262129:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201921.262134:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201921.262135:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201921.262141:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1052 00000100:00080000:2.0:1788201921.262146:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201921.262150:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201921.262156:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788201921.262162:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559777408:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201921.262173:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559777664/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201921.262240:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201921.262279:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201921/real 1788201921] req@ffff93c0e62b8a80 x1875064559777408/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201921.262310:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559777408/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201921.262319:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201921.262322:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201921.262326:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559777408:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201921.262332:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559777536:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201921.262347:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201921.262355:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201921/real 1788201921] req@ffff93c0e62bad80 x1875064559777536/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201921.262363:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559777536/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201921.262369:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201921.262371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201921.262373:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559777536:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201921.262393:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559777664:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201921.262405:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201921.262411:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201921/real 1788201921] req@ffff93c0e62bb100 x1875064559777664/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201921.262418:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559777664/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201921.262429:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201921.262431:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201921.262441:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559777664:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201921.965982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201922.989988:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201924.013982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201925.037985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788201925.870029:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201925.870048:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201925.870197:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201925.870273:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201925.870278:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201925.870308:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201926.062028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201926.381921:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201926.381931:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201926.381935:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201926.381944:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1057 00000100:00080000:3.0:1788201926.381955:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201926.381963:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788201926.382000:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559777792/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201926.382026:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201926.382028:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201926.382030:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201926.382033:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1057 00000100:00080000:3.0:1788201926.382035:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201926.382039:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201926.382048:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201926.382060:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559777792:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201926.382061:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559777920/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201926.382076:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201926.382079:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201926.382080:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201926.382083:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1057 00000100:00080000:3.0:1788201926.382085:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201926.382088:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201926.382092:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788201926.382112:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559778048/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201926.382154:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201926.382194:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201926/real 1788201926] req@ffff93c1c4a4c000 x1875064559777792/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201926.382242:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559777792/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201926.382252:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201926.382255:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201926.382258:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559777792:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201926.382279:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559777920:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201926.382312:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201926.382321:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201926/real 1788201926] req@ffff93c1c4a4e680 x1875064559777920/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201926.382330:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559777920/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201926.382336:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201926.382339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201926.382341:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559777920:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201926.382345:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559778048:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201926.382355:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201926.382361:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201926/real 1788201926] req@ffff93c1c4a4fb80 x1875064559778048/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201926.382369:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559778048/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201926.382380:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201926.382393:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201926.382396:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559778048:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201927.085950:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201928.110003:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201929.133962:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201930.157976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788201930.990144:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201930.990173:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201930.990289:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201930.990320:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201930.990324:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201930.990346:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201931.181960:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201931.501944:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201931.501956:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201931.501962:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201931.501973:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1062 00000100:00080000:3.0:1788201931.501985:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201931.501993:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788201931.502043:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559778176/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201931.502074:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201931.502078:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201931.502080:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201931.502090:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1062 00000100:00080000:3.0:1788201931.502093:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201931.502098:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201931.502122:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201931.502136:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559778304/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201931.502145:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201931.502148:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201931.502151:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788201931.502154:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559778176:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788201931.502156:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1062 00000100:00080000:3.0:1788201931.502163:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201931.502168:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201931.502172:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788201931.502182:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f100 x1875064559778432/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201931.502243:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201931.502283:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201931/real 1788201931] req@ffff93c1c4a4dc00 x1875064559778176/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201931.502311:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559778176/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201931.502323:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201931.502327:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201931.502332:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559778176:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201931.502356:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559778304:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201931.502379:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201931.502392:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201931/real 1788201931] req@ffff93c1c4a4ed80 x1875064559778304/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201931.502422:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559778304/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201931.502435:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201931.502439:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201931.502443:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559778304:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201931.502450:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559778432:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201931.502470:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201931.502480:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201931/real 1788201931] req@ffff93c1c4a4f100 x1875064559778432/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201931.502492:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f100 x1875064559778432/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201931.502502:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201931.502506:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201931.502509:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559778432:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201932.206004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201933.229929:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201934.254001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201935.277999:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788201936.110075:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201936.110097:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201936.110231:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201936.110255:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201936.110258:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201936.110276:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201936.302098:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201936.621984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201936.621994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201936.621998:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201936.622009:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1067 00000100:00080000:1.0:1788201936.622033:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201936.622043:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788201936.622091:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559778560/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201936.622118:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201936.622121:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201936.622122:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201936.622125:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1067 00000100:00080000:1.0:1788201936.622128:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201936.622132:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201936.622142:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201936.622152:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559778688/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201936.622161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201936.622164:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201936.622165:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201936.622169:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1067 00000100:00080000:1.0:1788201936.622171:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201936.622173:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201936.622178:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788201936.622186:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559778816/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201936.622199:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559778560:192.168.201.156@tcp:250:kworker.0 00000800:00000100:2.0:1788201936.622275:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201936.622324:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201936/real 1788201936] req@ffff93c1c5037800 x1875064559778560/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201936.622341:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559778560/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201936.622349:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201936.622353:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201936.622356:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559778560:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201936.622362:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559778688:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201936.622378:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201936.622387:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201936/real 1788201936] req@ffff93c1c5036680 x1875064559778688/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201936.622396:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559778688/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201936.622401:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201936.622403:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201936.622405:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559778688:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201936.622427:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559778816:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201936.622439:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201936.622445:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201936/real 1788201936] req@ffff93c1c5035c00 x1875064559778816/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201936.622453:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559778816/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201936.622460:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201936.622462:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201936.622464:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559778816:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201937.325942:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201938.350049:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201939.374084:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201940.398045:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788201941.230011:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201941.230026:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201941.230120:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201941.230144:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201941.230147:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201941.230166:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201941.422030:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201941.741957:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201941.741966:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201941.741970:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201941.741979:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1072 00000100:00080000:3.0:1788201941.741989:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201941.741996:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788201941.742035:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4df80 x1875064559778944/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201941.742064:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201941.742068:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201941.742070:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201941.742074:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1072 00000100:00080000:3.0:1788201941.742078:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201941.742082:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201941.742091:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201941.742105:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f100 x1875064559779072/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201941.742108:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559778944:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788201941.742122:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201941.742125:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201941.742127:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201941.742130:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1072 00000100:00080000:3.0:1788201941.742134:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201941.742136:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201941.742140:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788201941.742150:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559779200/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201941.742181:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201941.742215:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201941/real 1788201941] req@ffff93c1c4a4df80 x1875064559778944/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201941.742232:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4df80 x1875064559778944/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201941.742244:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201941.742247:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201941.742251:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559778944:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201941.742269:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559779072:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201941.742290:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201941.742304:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201941/real 1788201941] req@ffff93c1c4a4f100 x1875064559779072/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201941.742318:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f100 x1875064559779072/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201941.742328:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201941.742332:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201941.742335:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559779072:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201941.742342:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559779200:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201941.742358:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201941.742366:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201941/real 1788201941] req@ffff93c1c4a4ed80 x1875064559779200/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201941.742374:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559779200/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201941.742382:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201941.742384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201941.742386:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559779200:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201942.446006:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201943.470057:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201944.493987:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201945.518028:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201946.350118:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201946.350136:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201946.350271:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201946.350292:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201946.350295:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201946.350321:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201946.541978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201946.861955:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201946.861965:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201946.861970:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201946.861981:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1077 00000100:00080000:3.0:1788201946.861993:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201946.862000:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788201946.862043:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559779328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201946.862070:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201946.862075:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201946.862076:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201946.862081:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1077 00000100:00080000:3.0:1788201946.862085:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201946.862089:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201946.862104:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201946.862117:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559779456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201946.862128:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788201946.862130:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559779328:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788201946.862133:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201946.862135:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201946.862139:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1077 00000100:00080000:3.0:1788201946.862142:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201946.862146:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201946.862151:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788201946.862161:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559779584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201946.862201:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201946.862245:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201946/real 1788201946] req@ffff93c1c4a4dc00 x1875064559779328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201946.862260:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559779328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201946.862269:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201946.862271:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201946.862274:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559779328:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201946.862311:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559779456:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201946.862337:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201946.862349:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201946/real 1788201946] req@ffff93c1c4a4fb80 x1875064559779456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201946.862358:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559779456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201946.862364:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201946.862367:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201946.862369:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559779456:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201946.862373:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559779584:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201946.862387:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201946.862394:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201946/real 1788201946] req@ffff93c1c4a4e680 x1875064559779584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201946.862401:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559779584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201946.862409:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201946.862411:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201946.862413:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559779584:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201947.565986:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201948.589946:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201949.614002:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201950.637960:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788201951.470056:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201951.470073:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201951.470198:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201951.470226:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201951.470229:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201951.470247:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201951.662038:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201951.981930:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201951.981945:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201951.981951:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201951.981976:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1082 00000100:00080000:3.0:1788201951.982006:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201951.982016:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788201951.982094:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559779712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201951.982126:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201951.982130:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201951.982144:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201951.982149:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1082 00000100:00080000:3.0:1788201951.982153:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201951.982159:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201951.982171:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201951.982183:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559779712:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201951.982190:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559779840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201951.982205:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201951.982209:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201951.982211:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201951.982216:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1082 00000100:00080000:3.0:1788201951.982221:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201951.982225:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201951.982231:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788201951.982245:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559779968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201951.982256:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201951.982313:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201951/real 1788201951] req@ffff93c1c4a4c000 x1875064559779712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201951.982327:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559779712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201951.982335:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201951.982338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201951.982341:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559779712:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201951.982363:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559779840:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201951.982383:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201951.982408:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201951/real 1788201951] req@ffff93c1c4a4ea00 x1875064559779840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201951.982423:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559779840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201951.982433:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201951.982442:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201951.982446:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559779840:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201951.982453:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559779968:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201951.982478:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201951.982490:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201951/real 1788201951] req@ffff93c1c4a4c380 x1875064559779968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201951.982503:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559779968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201951.982515:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201951.982518:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201951.982522:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559779968:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201952.685980:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201953.710053:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201954.733933:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201955.758075:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201956.589971:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201956.589986:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201956.590075:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201956.590092:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201956.590095:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201956.590113:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201956.782072:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201957.101962:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201957.101971:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201957.101975:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201957.101984:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1087 00000100:00080000:2.0:1788201957.101995:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201957.102002:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201957.102038:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559780096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201957.102061:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201957.102064:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201957.102066:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201957.102069:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1087 00000100:00080000:2.0:1788201957.102071:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201957.102073:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201957.102088:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201957.102097:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559780224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201957.102106:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201957.102109:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201957.102111:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788201957.102113:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559780096:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788201957.102116:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1087 00000100:00080000:2.0:1788201957.102122:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201957.102126:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201957.102132:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201957.102141:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559780352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201957.102194:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201957.102237:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201957/real 1788201957] req@ffff93c0e62b9c00 x1875064559780096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201957.102259:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559780096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201957.102271:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201957.102276:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201957.102281:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559780096:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201957.102307:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559780224:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201957.102333:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201957.102348:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201957/real 1788201957] req@ffff93c0e62bbb80 x1875064559780224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201957.102362:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559780224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201957.102372:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201957.102376:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201957.102380:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559780224:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201957.102387:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559780352:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201957.102408:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201957.102419:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201957/real 1788201957] req@ffff93c0e62ba680 x1875064559780352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201957.102433:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559780352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201957.102444:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201957.102447:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201957.102451:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559780352:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201957.806061:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201958.829952:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201959.853994:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201960.878066:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201961.710026:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201961.710044:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201961.710138:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201961.710158:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201961.710162:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201961.710197:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201961.902015:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201962.222004:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201962.222014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201962.222018:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201962.222029:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1093 00000100:00080000:1.0:1788201962.222041:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201962.222048:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788201962.222094:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559780480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201962.222118:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201962.222120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201962.222122:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201962.222125:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1093 00000100:00080000:1.0:1788201962.222127:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201962.222130:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201962.222140:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201962.222150:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559780608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201962.222158:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201962.222161:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201962.222164:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788201962.222167:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559780480:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788201962.222169:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1093 00000100:00080000:1.0:1788201962.222177:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201962.222179:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201962.222183:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788201962.222192:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559780736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201962.222240:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201962.222288:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201962/real 1788201962] req@ffff93c1c5035c00 x1875064559780480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201962.222303:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559780480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201962.222312:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201962.222315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201962.222318:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559780480:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201962.222342:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559780608:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201962.222360:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201962.222370:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201962/real 1788201962] req@ffff93c1c5036680 x1875064559780608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201962.222384:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559780608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201962.222392:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201962.222396:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201962.222400:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559780608:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201962.222407:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559780736:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201962.222429:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201962.222438:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201962/real 1788201962] req@ffff93c1c5037800 x1875064559780736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201962.222455:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559780736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201962.222466:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201962.222470:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201962.222474:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559780736:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201962.926004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201963.949990:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201964.974006:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201965.997974:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788201966.830028:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201966.830045:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201966.830146:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201966.830175:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201966.830180:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201966.830208:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201967.021902:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201967.341937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201967.341947:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201967.341952:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201967.341960:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1098 00000100:00080000:1.0:1788201967.341976:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201967.341984:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788201967.342020:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559780864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201967.342045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201967.342051:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201967.342053:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201967.342056:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1098 00000100:00080000:1.0:1788201967.342058:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201967.342061:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201967.342070:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201967.342079:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559780992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201967.342088:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788201967.342091:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559780864:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788201967.342092:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201967.342095:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201967.342103:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1098 00000100:00080000:1.0:1788201967.342105:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201967.342107:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201967.342110:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788201967.342122:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559781120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788201967.342168:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201967.342203:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201967/real 1788201967] req@ffff93c1c5037100 x1875064559780864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201967.342217:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559780864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201967.342226:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201967.342229:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201967.342232:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559780864:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201967.342250:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559780992:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788201967.342264:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201967.342271:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201967/real 1788201967] req@ffff93c1c5034e00 x1875064559780992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201967.342279:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559780992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201967.342289:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201967.342291:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201967.342293:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559780992:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201967.342296:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559781120:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788201967.342307:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788201967.342312:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201967/real 1788201967] req@ffff93c1c5034a80 x1875064559781120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201967.342319:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559781120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201967.342326:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201967.342328:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201967.342329:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559781120:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201968.045942:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201969.069997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201970.093959:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201971.117996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788201971.950010:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201971.950026:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201971.950145:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788201971.950166:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788201971.950169:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788201971.950186:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201972.141933:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788201972.462007:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201972.462018:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788201972.462022:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201972.462033:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1103 00000100:00080000:2.0:1788201972.462045:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201972.462052:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201972.462095:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559781248/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201972.462124:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201972.462127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788201972.462129:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201972.462132:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1103 00000100:00080000:2.0:1788201972.462134:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201972.462137:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201972.462147:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788201972.462164:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559781376/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788201972.462173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788201972.462175:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788201972.462176:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788201972.462181:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1103 00000100:00080000:2.0:1788201972.462183:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788201972.462186:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788201972.462189:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788201972.462200:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559781504/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788201972.462204:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559781248:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201972.462336:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559781376:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788201972.462380:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559781504:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201973.165992:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201974.189931:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201975.214008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788201975.534155:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:0.0:1788201975.534207:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788201975.534226:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788201975.534278:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788201975.534291:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788201975.534337:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201972/real 1788201975] req@ffff93c0e62b8000 x1875064559781504/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201975.534385:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559781504/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788201997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201975.534414:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201975.534420:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201975.534425:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559781504:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788201975.534435:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201972/real 1788201975] req@ffff93c0e62b8380 x1875064559781248/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201975.534451:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559781248/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788201997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201975.534460:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201975.534464:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201975.534468:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559781248:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788201975.534475:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201972/real 1788201975] req@ffff93c0e62b8700 x1875064559781376/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788201975.534489:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559781376/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788201997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788201975.534497:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788201975.534501:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788201975.534504:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559781376:192.168.201.156@tcp:38:kworker.0 00010000:00010000:0.0:1788201976.238012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788201977.070062:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201977.070079:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201977.070202:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201977.070238:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201977.070244:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201977.070279:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201977.261994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201977.581971:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201977.581982:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201977.581986:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201977.581998:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1108 00000100:00080000:3.0:1788201977.582009:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201977.582017:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788201977.582054:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559781632/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201977.582082:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201977.582085:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201977.582086:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201977.582090:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1108 00000100:00080000:3.0:1788201977.582092:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201977.582094:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201977.582104:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201977.582114:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559781760/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201977.582123:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201977.582126:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201977.582127:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201977.582132:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1108 00000100:00080000:3.0:1788201977.582136:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788201977.582137:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559781632:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788201977.582144:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201977.582151:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788201977.582161:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559781888/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201977.582220:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201977.582270:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201977/real 1788201977] req@ffff93c1c4a4c700 x1875064559781632/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201977.582291:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559781632/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201977.582304:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201977.582326:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201977.582331:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559781632:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201977.582364:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559781760:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201977.582391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201977.582407:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201977/real 1788201977] req@ffff93c1c4a4f480 x1875064559781760/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201977.582421:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559781760/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201977.582430:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201977.582434:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201977.582437:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559781760:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201977.582444:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559781888:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201977.582467:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201977.582479:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201977/real 1788201977] req@ffff93c1c4a4e300 x1875064559781888/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201977.582492:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559781888/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201977.582503:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201977.582506:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201977.582508:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559781888:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201978.286006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201979.309999:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201980.333998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201981.358003:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788201982.190013:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201982.190029:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201982.190124:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201982.190156:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201982.190161:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201982.190197:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201982.382007:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201982.701972:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201982.701984:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201982.701990:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201982.702003:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1113 00000100:00080000:3.0:1788201982.702018:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201982.702026:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788201982.702083:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559782016/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201982.702117:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201982.702121:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201982.702123:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201982.702128:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1113 00000100:00080000:3.0:1788201982.702132:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201982.702137:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201982.702152:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201982.702167:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559782144/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201982.702181:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559782016:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788201982.702182:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201982.702193:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201982.702195:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201982.702201:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1113 00000100:00080000:3.0:1788201982.702204:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201982.702208:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201982.702214:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788201982.702262:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559782272/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201982.702285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201982.702335:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201982/real 1788201982] req@ffff93c1c4a4d880 x1875064559782016/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201982.702367:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559782016/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201982.702379:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201982.702383:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201982.702388:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559782016:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201982.702417:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559782144:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201982.702441:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201982.702454:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201982/real 1788201982] req@ffff93c1c4a4d180 x1875064559782144/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201982.702467:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559782144/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201982.702476:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201982.702479:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201982.702483:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559782144:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201982.702490:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559782272:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201982.702509:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201982.702520:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201982/real 1788201982] req@ffff93c1c4a4f800 x1875064559782272/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201982.702540:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559782272/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201982.702551:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201982.702554:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201982.702558:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559782272:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201983.405953:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201984.430006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201985.453963:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201986.477998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788201987.310002:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201987.310016:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201987.310101:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788201987.310124:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788201987.310127:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788201987.310171:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201987.501939:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788201987.821953:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201987.821963:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788201987.821966:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201987.821975:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1118 00000100:00080000:3.0:1788201987.821986:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201987.821992:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788201987.822027:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d500 x1875064559782400/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201987.822050:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201987.822053:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788201987.822054:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201987.822057:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1118 00000100:00080000:3.0:1788201987.822059:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201987.822062:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788201987.822069:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788201987.822077:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ca80 x1875064559782528/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788201987.822084:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788201987.822087:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788201987.822088:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788201987.822091:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1118 00000100:00080000:3.0:1788201987.822093:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788201987.822103:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788201987.822105:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559782400:192.168.201.156@tcp:250:kworker.0 00000008:00100000:3.0:1788201987.822107:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788201987.822121:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559782656/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201987.822199:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201987.822238:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201987/real 1788201987] req@ffff93c1c4a4d500 x1875064559782400/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201987.822253:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d500 x1875064559782400/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201987.822261:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201987.822264:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201987.822267:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559782400:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201987.822273:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559782528:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201987.822289:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201987.822297:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201987/real 1788201987] req@ffff93c1c4a4ca80 x1875064559782528/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201987.822305:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ca80 x1875064559782528/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201987.822314:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201987.822316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201987.822318:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559782528:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201987.822338:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559782656:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201987.822350:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201987.822356:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201987/real 1788201987] req@ffff93c1c4a4f800 x1875064559782656/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201987.822363:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559782656/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201987.822369:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201987.822371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201987.822373:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559782656:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201988.525974:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201989.549939:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201990.573953:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201991.597929:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788201992.430097:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201992.430111:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201992.430194:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201992.430215:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201992.430218:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201992.430235:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201992.621977:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201992.942004:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201992.942014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201992.942018:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201992.942029:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1123 00000100:00080000:1.0:1788201992.942054:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201992.942064:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788201992.942108:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559782784/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201992.942133:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201992.942136:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201992.942137:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201992.942140:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1123 00000100:00080000:1.0:1788201992.942143:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201992.942146:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201992.942153:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201992.942162:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559782912/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788201992.942175:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559782784:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788201992.942178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201992.942184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201992.942185:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201992.942189:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1123 00000100:00080000:1.0:1788201992.942192:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201992.942194:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201992.942197:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788201992.942206:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559783040/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201992.942241:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201992.942276:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201992/real 1788201992] req@ffff93c1c5034a80 x1875064559782784/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201992.942291:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559782784/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201992.942300:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201992.942303:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201992.942306:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559782784:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201992.942325:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559782912:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201992.942340:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201992.942348:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201992/real 1788201992] req@ffff93c1c5034e00 x1875064559782912/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201992.942357:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559782912/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201992.942364:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201992.942367:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201992.942371:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559782912:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201992.942378:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559783040:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201992.942398:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201992.942410:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201992/real 1788201992] req@ffff93c1c5037100 x1875064559783040/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201992.942423:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559783040/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201992.942432:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201992.942435:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201992.942437:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559783040:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201993.646009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201994.669954:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201995.694014:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201996.717937:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788201997.550035:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201997.550056:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201997.550230:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788201997.550270:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788201997.550277:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788201997.550311:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788201997.742040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788201998.061943:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201998.061968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788201998.061973:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201998.061983:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1128 00000100:00080000:1.0:1788201998.061996:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201998.062003:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788201998.062056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559783168/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201998.062098:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201998.062102:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788201998.062104:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201998.062108:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1128 00000100:00080000:1.0:1788201998.062113:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788201998.062118:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201998.062130:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788201998.062147:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559783296/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788201998.062157:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788201998.062161:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788201998.062162:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788201998.062166:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1128 00000100:00080000:1.0:1788201998.062173:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788201998.062176:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559783168:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788201998.062179:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788201998.062188:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788201998.062199:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559783424/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788201998.062266:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201998.062324:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201998/real 1788201998] req@ffff93c1c5037800 x1875064559783168/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201998.062348:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559783168/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201998.062361:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201998.062375:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201998.062380:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559783168:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788201998.062409:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559783296:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788201998.062435:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201998.062449:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201998/real 1788201998] req@ffff93c1c5036680 x1875064559783296/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201998.062463:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559783296/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201998.062473:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201998.062477:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201998.062481:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559783296:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788201998.062488:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559783424:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788201998.062508:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788201998.062520:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788201998/real 1788201998] req@ffff93c1c5035c00 x1875064559783424/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788201998.062545:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559783424/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788201998.062563:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788201998.062567:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788201998.062571:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559783424:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788201998.766050:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788201999.790018:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202000.813963:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202001.838054:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202002.670098:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202002.670134:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202002.670308:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202002.670356:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202002.670361:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202002.670389:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202002.862062:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202003.182058:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202003.182077:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202003.182083:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202003.182101:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1134 00000100:00080000:1.0:1788202003.182118:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202003.182127:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202003.182190:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559783552/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202003.182235:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202003.182239:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202003.182241:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202003.182246:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1134 00000100:00080000:1.0:1788202003.182250:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202003.182254:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202003.182266:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202003.182280:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559783680/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202003.182290:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202003.182294:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202003.182296:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202003.182300:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1134 00000100:00080000:1.0:1788202003.182302:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202003.182306:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202003.182311:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202003.182320:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559783808/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788202003.182415:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559783552:192.168.201.156@tcp:250:kworker.0 00000800:00000100:2.0:1788202003.182549:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202003.182617:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202003/real 1788202003] req@ffff93c1c5035c00 x1875064559783552/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202003.182658:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559783552/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202003.182674:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202003.182680:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202003.182686:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559783552:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202003.182698:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559783680:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202003.182731:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202003.182748:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202003/real 1788202003] req@ffff93c1c5036680 x1875064559783680/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202003.182770:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559783680/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202003.182787:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202003.182794:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202003.182801:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559783680:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202003.182816:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559783808:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202003.182948:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202003.182967:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202003/real 1788202003] req@ffff93c1c5037800 x1875064559783808/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202003.182984:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559783808/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202003.182997:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202003.183002:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202003.183006:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559783808:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202003.886041:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202004.909990:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202005.934057:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202006.957953:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202007.789993:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202007.790010:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202007.790124:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202007.790151:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202007.790156:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202007.790188:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202007.982008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202008.301909:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202008.301918:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202008.301922:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202008.301930:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1139 00000100:00080000:1.0:1788202008.301941:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202008.301948:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202008.301984:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559783936/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202008.302010:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202008.302014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202008.302017:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202008.302021:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1139 00000100:00080000:1.0:1788202008.302025:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202008.302029:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202008.302039:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202008.302052:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559784064/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202008.302063:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788202008.302063:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559783936:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202008.302067:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202008.302069:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202008.302073:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1139 00000100:00080000:1.0:1788202008.302077:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202008.302080:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202008.302085:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202008.302095:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559784192/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202008.302134:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202008.302174:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202008/real 1788202008] req@ffff93c1c5037100 x1875064559783936/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202008.302190:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559783936/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202008.302198:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202008.302201:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202008.302204:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559783936:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202008.302223:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559784064:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202008.302241:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202008.302249:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202008/real 1788202008] req@ffff93c1c5034e00 x1875064559784064/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202008.302257:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559784064/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202008.302263:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202008.302265:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202008.302268:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559784064:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202008.302271:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559784192:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202008.302281:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202008.302287:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202008/real 1788202008] req@ffff93c1c5034a80 x1875064559784192/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202008.302294:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559784192/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202008.302301:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202008.302303:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202008.302305:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559784192:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202009.005923:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202010.030026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202011.053976:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202012.077952:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202012.910017:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202012.910038:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202012.910176:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202012.910218:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202012.910224:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202012.910258:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202013.101943:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202013.421947:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202013.421956:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202013.421959:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202013.421968:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1144 00000100:00080000:2.0:1788202013.421979:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202013.421988:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202013.422031:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559784320/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202013.422057:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202013.422060:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202013.422062:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202013.422065:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1144 00000100:00080000:2.0:1788202013.422068:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202013.422071:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202013.422080:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202013.422091:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559784448/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202013.422098:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202013.422101:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202013.422102:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202013.422105:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1144 00000100:00080000:2.0:1788202013.422107:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202013.422111:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202013.422115:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559784320:192.168.201.156@tcp:250:kworker.0 00000008:00100000:2.0:1788202013.422116:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202013.422130:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559784576/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202013.422196:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202013.422241:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202013/real 1788202013] req@ffff93c0e62baa00 x1875064559784320/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202013.422261:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559784320/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202013.422272:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202013.422276:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202013.422280:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559784320:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202013.422288:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559784448:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202013.422311:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202013.422325:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202013/real 1788202013] req@ffff93c0e62bb480 x1875064559784448/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202013.422337:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559784448/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202013.422345:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202013.422349:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202013.422352:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559784448:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202013.422378:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559784576:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202013.422399:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202013.422412:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202013/real 1788202013] req@ffff93c0e62b8700 x1875064559784576/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202013.422425:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559784576/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202013.422439:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202013.422441:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202013.422444:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559784576:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202014.125997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202015.149988:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202016.173981:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202017.198030:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202018.030043:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202018.030059:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202018.030153:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202018.030175:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202018.030181:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202018.030217:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202018.222029:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202018.541898:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202018.541908:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202018.541912:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202018.541920:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1149 00000100:00080000:2.0:1788202018.541930:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202018.541937:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202018.541973:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559784704/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202018.542001:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202018.542004:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202018.542005:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202018.542009:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1149 00000100:00080000:2.0:1788202018.542011:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202018.542014:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202018.542025:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202018.542033:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559784832/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202018.542050:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202018.542053:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202018.542055:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202018.542060:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1149 00000100:00080000:2.0:1788202018.542065:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202018.542070:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202018.542076:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202018.542089:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559784704:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202018.542091:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559784960/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202018.542153:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202018.542196:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202018/real 1788202018] req@ffff93c0e62b8380 x1875064559784704/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202018.542212:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559784704/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202018.542221:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202018.542224:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202018.542227:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559784704:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202018.542232:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559784832:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202018.542256:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202018.542265:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202018/real 1788202018] req@ffff93c0e62b8000 x1875064559784832/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202018.542273:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559784832/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202018.542278:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202018.542280:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202018.542282:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559784832:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202018.542301:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559784960:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202018.542313:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202018.542319:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202018/real 1788202018] req@ffff93c0e62ba680 x1875064559784960/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202018.542326:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559784960/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202018.542333:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202018.542335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202018.542336:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559784960:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202019.245966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202020.269992:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202021.294038:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202022.317978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202023.150024:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202023.150038:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202023.150132:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202023.150150:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202023.150154:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202023.150173:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202023.341985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202023.661950:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202023.661960:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202023.661964:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202023.661973:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1154 00000100:00080000:0.0:1788202023.661983:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202023.661990:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202023.662029:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559785088/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202023.662051:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202023.662053:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202023.662055:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202023.662058:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1154 00000100:00080000:0.0:1788202023.662060:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202023.662064:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202023.662072:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202023.662085:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559785216/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202023.662094:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202023.662097:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202023.662098:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202023.662101:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1154 00000100:00080000:0.0:1788202023.662104:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202023.662110:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202023.662115:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202023.662115:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559785088:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202023.662127:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559785344/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202023.662210:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202023.662260:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202023/real 1788202023] req@ffff93c0e9ce4a80 x1875064559785088/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202023.662276:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559785088/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202023.662287:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202023.662290:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202023.662293:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559785088:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202023.662321:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559785216:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202023.662338:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202023.662347:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202023/real 1788202023] req@ffff93c0e9ce5500 x1875064559785216/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202023.662356:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559785216/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202023.662362:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202023.662364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202023.662366:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559785216:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202023.662371:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559785344:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202023.662382:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202023.662388:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202023/real 1788202023] req@ffff93c0e9ce7800 x1875064559785344/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202023.662406:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559785344/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202023.662413:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202023.662415:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202023.662417:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559785344:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202024.366042:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202025.390069:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202026.413984:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202027.438001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202028.270051:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202028.270074:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202028.270192:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202028.270216:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202028.270219:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202028.270244:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202028.461948:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202028.781981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202028.781993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202028.781999:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202028.782010:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1159 00000100:00080000:2.0:1788202028.782023:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202028.782033:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202028.782073:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559785472/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202028.782103:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202028.782105:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202028.782107:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202028.782113:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1159 00000100:00080000:2.0:1788202028.782116:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202028.782119:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202028.782127:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202028.782139:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559785600/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202028.782151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788202028.782156:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559785472:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202028.782157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202028.782162:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202028.782170:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1159 00000100:00080000:2.0:1788202028.782174:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202028.782179:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202028.782184:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202028.782198:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559785728/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202028.782236:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202028.782277:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202028/real 1788202028] req@ffff93c0e62bbb80 x1875064559785472/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202028.782293:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559785472/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202028.782306:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202028.782309:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202028.782312:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559785472:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202028.782330:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559785600:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202028.782347:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202028.782355:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202028/real 1788202028] req@ffff93c0e62b9c00 x1875064559785600/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202028.782363:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559785600/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202028.782368:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202028.782370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202028.782373:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559785600:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202028.782376:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559785728:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202028.782387:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202028.782393:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202028/real 1788202028] req@ffff93c0e62bb100 x1875064559785728/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202028.782400:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559785728/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202028.782407:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202028.782409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202028.782410:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559785728:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202029.486023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202030.509999:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202031.533976:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202032.558013:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202033.390020:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202033.390039:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202033.390150:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202033.390194:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202033.390200:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202033.390245:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202033.581953:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202033.901982:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202033.901991:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202033.901995:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202033.902004:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1164 00000100:00080000:2.0:1788202033.902014:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202033.902021:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202033.902063:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559785856/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202033.902096:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202033.902099:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202033.902101:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202033.902104:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1164 00000100:00080000:2.0:1788202033.902107:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202033.902111:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202033.902119:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202033.902130:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559785984/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202033.902138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202033.902141:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202033.902143:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202033.902146:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1164 00000100:00080000:2.0:1788202033.902149:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202033.902151:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202033.902155:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202033.902164:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559785856:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202033.902165:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559786112/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202033.902236:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202033.902287:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202033/real 1788202033] req@ffff93c0e62bad80 x1875064559785856/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202033.902311:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559785856/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202033.902320:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202033.902324:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202033.902327:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559785856:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202033.902345:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559785984:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202033.902371:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202033.902381:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202033/real 1788202033] req@ffff93c0e62b8a80 x1875064559785984/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202033.902390:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559785984/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202033.902396:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202033.902399:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202033.902401:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559785984:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202033.902422:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559786112:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202033.902435:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202033.902441:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202033/real 1788202033] req@ffff93c0e62b9f80 x1875064559786112/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202033.902450:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559786112/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202033.902456:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202033.902458:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202033.902460:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559786112:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202034.605984:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202035.630031:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202036.654044:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202037.677995:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202038.510011:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202038.510028:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202038.510136:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202038.510163:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202038.510168:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202038.510198:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202038.701999:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202039.021939:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202039.021952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202039.021957:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202039.021968:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1169 00000100:00080000:2.0:1788202039.021981:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202039.021990:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202039.022031:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559786240/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202039.022068:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202039.022072:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202039.022073:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202039.022078:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1169 00000100:00080000:2.0:1788202039.022082:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202039.022086:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202039.022096:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202039.022112:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559786368/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202039.022125:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788202039.022129:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559786240:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202039.022130:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202039.022134:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202039.022141:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1169 00000100:00080000:2.0:1788202039.022145:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202039.022150:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202039.022156:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202039.022171:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559786496/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202039.022263:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559786368:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202039.022299:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559786496:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202039.726016:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202040.750057:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202041.773974:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788202042.094216:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788202042.094270:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788202042.094288:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202042.094342:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202042.094354:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788202042.094404:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202039/real 1788202042] req@ffff93c0e62bb800 x1875064559786368/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202042.094436:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559786368/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202042.094452:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202042.094458:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202042.094463:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559786368:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202042.094474:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202039/real 1788202042] req@ffff93c0e62b9f80 x1875064559786496/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202042.094496:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559786496/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202042.094514:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202042.094533:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202042.094538:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559786496:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788202042.094547:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202039/real 1788202042] req@ffff93c0e62b9500 x1875064559786240/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202042.094561:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559786240/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202042.094570:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202042.094574:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202042.094577:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559786240:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788202042.798077:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202043.630035:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202043.630050:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202043.630147:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202043.630184:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202043.630193:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202043.630216:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202043.821956:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202044.142018:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202044.142026:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202044.142030:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202044.142039:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1175 00000100:00080000:1.0:1788202044.142049:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202044.142058:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202044.142091:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559786624/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202044.142112:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202044.142115:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202044.142116:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202044.142119:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1175 00000100:00080000:1.0:1788202044.142122:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202044.142124:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202044.142132:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202044.142140:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559786752/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202044.142148:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202044.142151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202044.142152:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202044.142167:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1175 00000100:00080000:1.0:1788202044.142171:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202044.142173:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202044.142178:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202044.142182:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559786624:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788202044.142192:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559786880/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202044.142265:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202044.142310:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202044/real 1788202044] req@ffff93c1c5034a80 x1875064559786624/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202044.142328:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559786624/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202044.142341:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202044.142345:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202044.142362:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559786624:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202044.142371:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559786752:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202044.142399:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202044.142414:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202044/real 1788202044] req@ffff93c1c5034e00 x1875064559786752/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202044.142439:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559786752/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202044.142452:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202044.142457:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202044.142460:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559786752:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202044.142486:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559786880:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202044.142510:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202044.142524:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202044/real 1788202044] req@ffff93c1c5037100 x1875064559786880/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202044.142536:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559786880/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202044.142546:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202044.142550:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202044.142553:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559786880:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202044.846020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202045.869956:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202046.894020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202047.917987:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202048.750105:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202048.750121:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202048.750237:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202048.750261:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202048.750264:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202048.750287:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202048.942019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202049.262000:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202049.262009:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202049.262013:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202049.262022:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1180 00000100:00080000:0.0:1788202049.262039:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202049.262046:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202049.262097:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559787008/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202049.262127:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202049.262132:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202049.262134:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202049.262140:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1180 00000100:00080000:0.0:1788202049.262144:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202049.262148:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202049.262164:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202049.262176:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559787136/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202049.262185:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788202049.262187:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559787008:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202049.262189:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202049.262191:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202049.262199:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1180 00000100:00080000:0.0:1788202049.262201:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202049.262204:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202049.262208:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202049.262216:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559787264/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202049.262282:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202049.262327:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202049/real 1788202049] req@ffff93c0e9ce5180 x1875064559787008/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202049.262366:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559787008/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202049.262377:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202049.262382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202049.262386:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559787008:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202049.262425:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559787136:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202049.262452:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202049.262466:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202049/real 1788202049] req@ffff93c0e9ce4e00 x1875064559787136/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202049.262479:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559787136/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202049.262488:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202049.262491:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202049.262495:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559787136:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202049.262502:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559787264:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202049.262523:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202049.262535:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202049/real 1788202049] req@ffff93c0e9ce5880 x1875064559787264/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202049.262547:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559787264/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202049.262564:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202049.262567:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202049.262571:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559787264:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202049.966016:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202050.990029:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202052.014003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202053.038033:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202053.870044:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202053.870060:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202053.870177:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202053.870212:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202053.870216:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202053.870239:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202054.062040:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202054.382038:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202054.382053:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202054.382058:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202054.382069:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1185 00000100:00080000:2.0:1788202054.382081:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202054.382088:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202054.382134:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559787392/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202054.382158:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202054.382161:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202054.382162:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202054.382166:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1185 00000100:00080000:2.0:1788202054.382168:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202054.382172:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202054.382183:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202054.382193:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559787392:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202054.382195:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559787520/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202054.382206:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202054.382209:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202054.382210:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202054.382214:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1185 00000100:00080000:2.0:1788202054.382216:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202054.382218:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202054.382222:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202054.382229:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559787648/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202054.382267:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202054.382317:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202054/real 1788202054] req@ffff93c0e62b8a80 x1875064559787392/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202054.382333:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559787392/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202054.382343:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202054.382346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202054.382349:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559787392:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202054.382373:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559787520:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202054.382395:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202054.382403:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202054/real 1788202054] req@ffff93c0e62bad80 x1875064559787520/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202054.382412:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559787520/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202054.382418:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202054.382420:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202054.382423:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559787520:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202054.382428:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559787648:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202054.382439:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202054.382445:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202054/real 1788202054] req@ffff93c0e62bb100 x1875064559787648/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202054.382453:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559787648/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202054.382460:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202054.382462:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202054.382464:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559787648:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202055.086031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202056.110059:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202057.134050:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202058.157996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202058.990058:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202058.990075:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202058.990191:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202058.990214:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202058.990218:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202058.990239:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202059.182003:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202059.502020:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202059.502032:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202059.502036:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202059.502048:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1190 00000100:00080000:0.0:1788202059.502061:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202059.502068:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202059.502117:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559787776/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202059.502152:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202059.502156:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202059.502158:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202059.502164:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1190 00000100:00080000:0.0:1788202059.502169:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202059.502173:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202059.502183:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202059.502205:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559787904/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202059.502214:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202059.502217:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202059.502219:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202059.502223:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1190 00000100:00080000:0.0:1788202059.502226:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202059.502229:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202059.502234:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202059.502242:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559788032/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202059.502255:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559787776:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788202059.502340:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202059.502393:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202059/real 1788202059] req@ffff93c0e9ce6300 x1875064559787776/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202059.502415:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559787776/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202059.502428:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202059.502433:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202059.502438:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559787776:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202059.502448:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559787904:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202059.502474:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202059.502490:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202059/real 1788202059] req@ffff93c0e9ce5880 x1875064559787904/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202059.502506:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559787904/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202059.502517:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202059.502521:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202059.502526:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559787904:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202059.502695:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559788032:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202059.502723:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202059.502737:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202059/real 1788202059] req@ffff93c0e9ce4e00 x1875064559788032/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202059.502751:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559788032/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202059.502763:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202059.502767:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202059.502771:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559788032:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202060.206031:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202061.229994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202062.253998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202063.277970:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202064.110068:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202064.110083:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202064.110183:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202064.110207:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202064.110210:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202064.110241:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202064.301973:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202064.621913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202064.621922:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202064.621926:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202064.621937:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1195 00000100:00080000:2.0:1788202064.621952:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202064.621960:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202064.622010:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559788160/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202064.622038:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202064.622042:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202064.622044:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202064.622049:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1195 00000100:00080000:2.0:1788202064.622055:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202064.622061:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202064.622076:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202064.622087:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559788160:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202064.622097:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559788288/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202064.622108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202064.622112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202064.622113:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202064.622119:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1195 00000100:00080000:2.0:1788202064.622122:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202064.622125:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202064.622130:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202064.622139:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559788416/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202064.622160:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202064.622207:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202064/real 1788202064] req@ffff93c0e62b9c00 x1875064559788160/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202064.622223:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559788160/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202064.622238:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202064.622243:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202064.622248:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559788160:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202064.622276:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559788288:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202064.622303:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202064.622318:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202064/real 1788202064] req@ffff93c0e62bbb80 x1875064559788288/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202064.622333:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559788288/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202064.622342:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202064.622346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202064.622350:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559788288:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202064.622356:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559788416:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202064.622376:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202064.622388:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202064/real 1788202064] req@ffff93c0e62ba680 x1875064559788416/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202064.622402:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559788416/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202064.622413:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202064.622417:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202064.622420:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559788416:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202065.325989:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202066.349978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202067.373955:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202068.397978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202069.229972:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202069.229986:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202069.230069:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202069.230096:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202069.230099:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202069.230118:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202069.422019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202069.741948:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202069.741958:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202069.741962:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202069.741973:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1200 00000100:00080000:2.0:1788202069.741984:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202069.741992:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202069.742043:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559788544/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202069.742081:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202069.742086:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202069.742088:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202069.742094:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1200 00000100:00080000:2.0:1788202069.742099:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202069.742103:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202069.742115:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202069.742128:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559788672/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202069.742137:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202069.742140:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202069.742142:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202069.742147:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1200 00000100:00080000:2.0:1788202069.742152:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202069.742161:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559788544:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202069.742162:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202069.742181:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202069.742203:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559788800/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202069.742249:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202069.742296:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202069/real 1788202069] req@ffff93c0e62b8000 x1875064559788544/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202069.742315:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559788544/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202069.742324:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202069.742327:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202069.742338:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559788544:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202069.742344:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559788672:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202069.742362:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202069.742371:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202069/real 1788202069] req@ffff93c0e62b8380 x1875064559788672/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202069.742379:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559788672/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202069.742384:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202069.742387:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202069.742389:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559788672:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202069.742410:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559788800:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202069.742421:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202069.742427:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202069/real 1788202069] req@ffff93c0e62b8700 x1875064559788800/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202069.742435:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559788800/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202069.742442:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202069.742444:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202069.742446:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559788800:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202070.445981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202071.470119:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202072.494015:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202073.518053:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202074.350214:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202074.350230:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202074.350331:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202074.350360:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202074.350363:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202074.350388:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202074.542003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202074.861964:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202074.861974:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202074.861979:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202074.861989:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1205 00000100:00080000:0.0:1788202074.861998:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202074.862004:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202074.862062:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559788928/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202074.862084:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202074.862087:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202074.862088:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202074.862090:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1205 00000100:00080000:0.0:1788202074.862093:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202074.862095:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202074.862109:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202074.862117:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559788928:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202074.862120:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559789056/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202074.862129:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202074.862131:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202074.862132:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202074.862135:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1205 00000100:00080000:0.0:1788202074.862138:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202074.862140:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202074.862143:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202074.862151:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559789184/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202074.862175:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202074.862220:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202074/real 1788202074] req@ffff93c0e9ce5180 x1875064559788928/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202074.862233:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559788928/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202074.862241:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202074.862244:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202074.862248:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559788928:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202074.862272:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559789056:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202074.862285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202074.862293:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202074/real 1788202074] req@ffff93c0e9ce7800 x1875064559789056/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202074.862304:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559789056/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202074.862318:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202074.862321:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202074.862324:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559789056:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202074.862330:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559789184:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202074.862348:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202074.862356:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202074/real 1788202074] req@ffff93c0e9ce5500 x1875064559789184/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202074.862369:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559789184/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202074.862376:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202074.862378:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202074.862380:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559789184:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202075.565938:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202076.589970:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202077.613990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202078.637992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202079.470088:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202079.470104:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202079.470318:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202079.470363:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202079.470369:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202079.470412:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202079.661996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202079.981966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202079.981979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202079.981983:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202079.981995:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1210 00000100:00080000:1.0:1788202079.982007:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202079.982014:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202079.982061:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559789312/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202079.982081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202079.982083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202079.982084:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202079.982087:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1210 00000100:00080000:1.0:1788202079.982089:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202079.982092:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202079.982100:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202079.982109:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559789440/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202079.982116:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202079.982119:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202079.982120:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202079.982123:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1210 00000100:00080000:1.0:1788202079.982125:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202079.982127:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202079.982131:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202079.982137:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559789312:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788202079.982139:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559789568/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202079.982322:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202079.982375:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202079/real 1788202079] req@ffff93c1c5037800 x1875064559789312/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202079.982396:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559789312/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202079.982406:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202079.982411:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202079.982414:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559789312:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202079.982424:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559789440:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202079.982469:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202079.982480:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202079/real 1788202079] req@ffff93c1c5036680 x1875064559789440/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202079.982487:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559789440/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202079.982493:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202079.982495:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202079.982497:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559789440:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202079.982516:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559789568:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202079.982553:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202079.982563:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202079/real 1788202079] req@ffff93c1c5035c00 x1875064559789568/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202079.982570:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559789568/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202079.982576:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202079.982578:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202079.982580:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559789568:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202080.685973:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202081.709988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202082.734059:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202083.758015:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202084.590048:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202084.590062:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202084.590145:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202084.590175:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202084.590179:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202084.590208:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202084.781956:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202085.101939:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202085.101949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202085.101953:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202085.101961:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1215 00000100:00080000:1.0:1788202085.101971:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202085.101978:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202085.102010:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559789696/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202085.102040:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202085.102042:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202085.102043:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202085.102046:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1215 00000100:00080000:1.0:1788202085.102048:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202085.102050:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202085.102057:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202085.102096:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559789824/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202085.102108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202085.102111:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202085.102112:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202085.102115:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1215 00000100:00080000:1.0:1788202085.102118:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202085.102122:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559789696:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202085.102123:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202085.102127:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202085.102135:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559789952/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202085.102185:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202085.102216:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202085/real 1788202085] req@ffff93c1c5035c00 x1875064559789696/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202085.102247:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559789696/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202085.102255:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202085.102257:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202085.102260:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559789696:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202085.102266:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559789824:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202085.102284:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202085.102296:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202085/real 1788202085] req@ffff93c1c5036680 x1875064559789824/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202085.102308:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559789824/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202085.102317:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202085.102320:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202085.102323:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559789824:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202085.102343:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559789952:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202085.102357:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202085.102376:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202085/real 1788202085] req@ffff93c1c5037800 x1875064559789952/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202085.102384:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559789952/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202085.102395:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202085.102397:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202085.102399:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559789952:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202085.806036:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202086.829946:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202087.854043:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202088.877955:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202089.709994:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202089.710009:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202089.710086:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202089.710104:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202089.710106:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202089.710121:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202089.901967:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202090.221950:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202090.221958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202090.221962:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202090.221969:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1221 00000100:00080000:1.0:1788202090.221979:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202090.221985:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202090.222017:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559790080/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202090.222042:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202090.222044:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202090.222045:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202090.222048:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1221 00000100:00080000:1.0:1788202090.222050:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202090.222052:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202090.222059:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202090.222066:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559790208/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202090.222072:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202090.222074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202090.222076:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202090.222078:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1221 00000100:00080000:1.0:1788202090.222079:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202090.222082:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202090.222084:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202090.222091:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559790336/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202090.222091:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559790080:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788202090.222148:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202090.222175:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202090/real 1788202090] req@ffff93c1c5037100 x1875064559790080/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202090.222187:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559790080/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202090.222194:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202090.222196:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202090.222199:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559790080:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202090.222203:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559790208:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202090.222215:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202090.222222:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202090/real 1788202090] req@ffff93c1c5034e00 x1875064559790208/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202090.222228:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559790208/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202090.222234:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202090.222235:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202090.222237:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559790208:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202090.222280:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559790336:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202090.222296:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202090.222307:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202090/real 1788202090] req@ffff93c1c5034a80 x1875064559790336/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202090.222318:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559790336/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202090.222327:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202090.222330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202090.222334:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559790336:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202090.925970:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202091.949988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202092.974003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202093.997962:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202094.830014:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202094.830030:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202094.830119:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202094.830142:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202094.830146:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202094.830170:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202095.021893:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202095.341936:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202095.341948:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202095.341952:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202095.341960:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1226 00000100:00080000:0.0:1788202095.341969:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202095.341975:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202095.342007:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559790464/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202095.342027:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202095.342030:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202095.342031:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202095.342034:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1226 00000100:00080000:0.0:1788202095.342036:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202095.342038:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202095.342045:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202095.342053:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559790592/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202095.342059:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202095.342061:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202095.342062:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202095.342064:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1226 00000100:00080000:0.0:1788202095.342066:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202095.342068:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202095.342071:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202095.342079:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559790720/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202095.342094:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559790464:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788202095.342151:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202095.342181:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202095/real 1788202095] req@ffff93c0e9ce4a80 x1875064559790464/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202095.342193:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559790464/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202095.342201:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202095.342203:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202095.342206:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559790464:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202095.342210:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559790592:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202095.342222:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202095.342228:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202095/real 1788202095] req@ffff93c0e9ce5f80 x1875064559790592/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202095.342236:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559790592/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202095.342240:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202095.342242:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202095.342244:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559790592:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202095.342281:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559790720:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202095.342300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202095.342311:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202095/real 1788202095] req@ffff93c0e9ce7100 x1875064559790720/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202095.342322:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559790720/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202095.342332:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202095.342335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202095.342337:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559790720:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202096.046041:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202097.070050:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202098.093955:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202099.117976:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202099.950040:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202099.951026:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202099.951111:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202099.951128:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202099.951130:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202099.951145:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202100.141961:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202100.461995:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202100.462006:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202100.462011:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202100.462021:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1231 00000100:00080000:2.0:1788202100.462032:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202100.462041:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202100.462081:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559790848/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202100.462111:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202100.462115:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202100.462117:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202100.462121:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1231 00000100:00080000:2.0:1788202100.462125:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202100.462129:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202100.462139:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202100.462153:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559790976/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202100.462171:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202100.462189:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202100.462191:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202100.462196:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1231 00000100:00080000:2.0:1788202100.462200:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202100.462204:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202100.462210:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202100.462225:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559790848:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202100.462226:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559791104/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202100.462320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202100.462383:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202100/real 1788202100] req@ffff93c0e62bb480 x1875064559790848/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202100.462406:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559790848/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202100.462425:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202100.462429:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202100.462434:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559790848:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202100.462449:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559790976:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202100.462473:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202100.462487:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202100/real 1788202100] req@ffff93c0e62baa00 x1875064559790976/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202100.462510:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559790976/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202100.462519:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202100.462523:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202100.462526:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559790976:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202100.462559:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559791104:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202100.462583:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202100.462595:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202100/real 1788202100] req@ffff93c0e62b8700 x1875064559791104/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202100.462607:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559791104/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202100.462618:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202100.462622:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202100.462625:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559791104:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202101.165994:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202102.189953:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202103.214018:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202104.237949:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202105.070009:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202105.070025:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202105.070310:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202105.070375:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202105.070387:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202105.070424:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202105.261973:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202105.581950:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202105.581961:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202105.581965:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202105.581975:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1236 00000100:00080000:2.0:1788202105.581987:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202105.581995:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202105.582035:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559791232/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202105.582062:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202105.582065:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202105.582066:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202105.582070:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1236 00000100:00080000:2.0:1788202105.582072:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202105.582074:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202105.582084:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202105.582110:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559791360/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202105.582124:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202105.582128:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202105.582130:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202105.582138:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1236 00000100:00100000:3.0:1788202105.582141:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559791232:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202105.582143:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202105.582152:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202105.582159:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202105.582171:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559791488/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202105.582267:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559791360:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202105.582289:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559791488:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202106.285931:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202107.310023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202108.333970:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788202108.654103:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788202108.654142:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788202108.654156:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202108.654185:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202108.654191:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788202108.654238:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202105/real 1788202108] req@ffff93c0e62b8000 x1875064559791360/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202108.654265:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559791360/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202108.654277:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202108.654282:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202108.654286:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559791360:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202108.654294:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202105/real 1788202108] req@ffff93c0e62ba680 x1875064559791488/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202108.654306:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559791488/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202108.654315:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202108.654319:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202108.654322:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559791488:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788202108.654328:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202105/real 1788202108] req@ffff93c0e62b8380 x1875064559791232/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202108.654340:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559791232/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202108.654372:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202108.654375:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202108.654379:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559791232:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788202109.357988:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202110.190126:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202110.190150:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202110.190270:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202110.190293:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202110.190296:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202110.190316:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202110.381978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202110.701972:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202110.701983:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202110.701986:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202110.702017:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1241 00000100:00080000:0.0:1788202110.702029:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202110.702038:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202110.702082:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559791616/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202110.702112:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202110.702116:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202110.702118:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202110.702129:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1241 00000100:00080000:0.0:1788202110.702134:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202110.702139:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202110.702152:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202110.702169:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559791744/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202110.702176:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559791616:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202110.702189:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202110.702194:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202110.702196:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202110.702201:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1241 00000100:00080000:0.0:1788202110.702205:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202110.702208:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202110.702213:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202110.702228:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559791872/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202110.702402:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202110.702459:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202110/real 1788202110] req@ffff93c0e9ce6d80 x1875064559791616/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202110.702481:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559791616/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202110.702491:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202110.702496:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202110.702500:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559791616:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202110.702532:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559791744:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202110.702565:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202110.702578:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202110/real 1788202110] req@ffff93c0e9ce5c00 x1875064559791744/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202110.702588:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559791744/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202110.702596:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202110.702599:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202110.702602:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559791744:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202110.702608:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559791872:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202110.702625:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202110.702634:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202110/real 1788202110] req@ffff93c0e9ce7b80 x1875064559791872/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202110.702644:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559791872/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202110.702653:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202110.702656:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202110.702659:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559791872:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202111.406005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202112.430001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202113.454033:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202114.478007:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202115.310014:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202115.310028:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202115.310111:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202115.310130:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202115.310133:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202115.310149:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202115.502035:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202115.821991:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202115.822002:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202115.822006:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202115.822016:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1246 00000100:00080000:0.0:1788202115.822028:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202115.822040:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202115.822082:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559792000/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202115.822103:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202115.822106:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202115.822107:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202115.822111:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1246 00000100:00080000:0.0:1788202115.822113:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202115.822115:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202115.822126:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202115.822135:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559792128/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202115.822143:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202115.822146:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202115.822147:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202115.822150:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1246 00000100:00080000:0.0:1788202115.822152:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202115.822155:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202115.822159:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202115.822167:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559792256/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202115.822172:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559792000:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788202115.822247:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202115.822295:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202115/real 1788202115] req@ffff93c0e9ce6680 x1875064559792000/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202115.822316:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559792000/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202115.822328:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202115.822332:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202115.822337:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559792000:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202115.822345:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559792128:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202115.822372:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202115.822387:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202115/real 1788202115] req@ffff93c0e9ce4000 x1875064559792128/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202115.822400:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559792128/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202115.822409:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202115.822413:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202115.822416:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559792128:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202115.822447:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559792256:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202115.822471:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202115.822483:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202115/real 1788202115] req@ffff93c0e9ce6a00 x1875064559792256/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202115.822497:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559792256/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202115.822508:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202115.822512:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202115.822515:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559792256:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202116.526012:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202117.550036:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202118.573936:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202119.597980:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202120.430027:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202120.430044:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202120.430145:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202120.430173:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202120.430179:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202120.430211:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202120.621987:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202120.941927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202120.941935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202120.941940:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202120.941955:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1251 00000100:00080000:1.0:1788202120.941965:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202120.941971:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202120.942007:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559792384/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202120.942035:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202120.942039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202120.942041:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202120.942045:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1251 00000100:00080000:1.0:1788202120.942049:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202120.942053:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202120.942063:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202120.942076:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559792512/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202120.942076:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559792384:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202120.942087:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202120.942090:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202120.942092:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202120.942096:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1251 00000100:00080000:1.0:1788202120.942099:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202120.942102:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202120.942106:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202120.942115:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559792640/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202120.942140:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202120.942179:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202120/real 1788202120] req@ffff93c1c5034a80 x1875064559792384/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202120.942194:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559792384/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202120.942205:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202120.942208:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202120.942211:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559792384:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202120.942230:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559792512:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202120.942247:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202120.942256:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202120/real 1788202120] req@ffff93c1c5034e00 x1875064559792512/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202120.942264:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559792512/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202120.942269:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202120.942271:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202120.942273:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559792512:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202120.942277:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559792640:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202120.942287:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202120.942293:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202120/real 1788202120] req@ffff93c1c5037100 x1875064559792640/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202120.942300:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559792640/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202120.942307:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202120.942308:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202120.942310:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559792640:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202121.645950:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202122.669982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202123.693978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202124.718001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202125.550019:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202125.550034:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202125.550126:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202125.550145:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202125.550148:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202125.550165:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202125.741947:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202126.061995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202126.062006:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202126.062012:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202126.062022:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1256 00000100:00080000:2.0:1788202126.062033:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202126.062042:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202126.062094:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559792768/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202126.062124:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202126.062128:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202126.062130:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202126.062135:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1256 00000100:00080000:2.0:1788202126.062139:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202126.062144:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202126.062154:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202126.062179:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559792768:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202126.062180:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559792896/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202126.062198:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202126.062202:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202126.062204:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202126.062208:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1256 00000100:00080000:2.0:1788202126.062212:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202126.062215:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202126.062220:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202126.062238:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559793024/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202126.062246:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202126.062292:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202126/real 1788202126] req@ffff93c0e62bbb80 x1875064559792768/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202126.062313:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559792768/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202126.062324:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202126.062327:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202126.062329:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559792768:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202126.062350:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559792896:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202126.062370:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202126.062379:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202126/real 1788202126] req@ffff93c0e62b9c00 x1875064559792896/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202126.062387:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559792896/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202126.062393:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202126.062395:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202126.062397:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559792896:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202126.062401:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559793024:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202126.062411:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202126.062417:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202126/real 1788202126] req@ffff93c0e62bb100 x1875064559793024/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202126.062425:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559793024/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202126.062431:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202126.062433:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202126.062435:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559793024:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202126.765996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202127.790042:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202128.814017:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202129.837975:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202130.670001:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202130.670020:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202130.670112:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202130.670133:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202130.670136:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202130.670154:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202130.861993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202131.181988:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202131.182000:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202131.182005:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202131.182017:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1262 00000100:00080000:2.0:1788202131.182028:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202131.182037:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202131.182078:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559793152/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202131.182110:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202131.182114:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202131.182117:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202131.182122:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1262 00000100:00080000:2.0:1788202131.182126:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202131.182130:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202131.182140:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202131.182154:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559793280/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202131.182164:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559793152:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202131.182168:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202131.182176:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202131.182178:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202131.182183:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1262 00000100:00080000:2.0:1788202131.182187:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202131.182192:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202131.182197:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202131.182211:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559793408/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202131.182246:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202131.182287:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202131/real 1788202131] req@ffff93c0e62bad80 x1875064559793152/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202131.182308:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559793152/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202131.182320:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202131.182325:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202131.182329:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559793152:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202131.182350:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559793280:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202131.182369:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202131.182378:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202131/real 1788202131] req@ffff93c0e62b8a80 x1875064559793280/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202131.182386:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559793280/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202131.182396:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202131.182398:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202131.182401:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559793280:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202131.182404:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559793408:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202131.182415:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202131.182420:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202131/real 1788202131] req@ffff93c0e62b9500 x1875064559793408/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202131.182427:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559793408/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202131.182434:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202131.182436:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202131.182438:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559793408:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202131.885927:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202132.909928:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202133.933992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202134.958015:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202135.790016:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202135.790031:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202135.790129:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202135.790158:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202135.790164:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202135.790197:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202135.981995:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202136.301899:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202136.301909:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202136.301914:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202136.301922:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1267 00000100:00080000:2.0:1788202136.301932:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202136.301939:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202136.301976:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559793536/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202136.301999:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202136.302001:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202136.302003:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202136.302006:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1267 00000100:00080000:2.0:1788202136.302008:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202136.302010:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202136.302019:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202136.302028:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559793664/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202136.302041:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788202136.302042:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559793536:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202136.302045:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202136.302049:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202136.302053:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1267 00000100:00080000:2.0:1788202136.302055:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202136.302058:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202136.302062:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202136.302070:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559793792/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202136.302107:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202136.302140:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202136/real 1788202136] req@ffff93c0e62b9f80 x1875064559793536/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202136.302161:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559793536/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202136.302169:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202136.302172:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202136.302175:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559793536:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202136.302196:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559793664:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202136.302212:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202136.302220:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202136/real 1788202136] req@ffff93c0e62bb800 x1875064559793664/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202136.302229:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559793664/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202136.302234:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202136.302237:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202136.302239:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559793664:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202136.302243:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559793792:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202136.302253:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202136.302259:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202136/real 1788202136] req@ffff93c0e62b9500 x1875064559793792/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202136.302273:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559793792/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202136.302280:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202136.302282:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202136.302285:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559793792:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202137.005989:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202138.029990:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202139.054036:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202140.077947:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202140.909976:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202140.909990:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202140.910088:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202140.910118:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202140.910125:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202140.910164:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202141.101957:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202141.421938:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202141.421948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202141.421951:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202141.421959:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1272 00000100:00080000:1.0:1788202141.421969:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202141.421975:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202141.422007:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559793920/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202141.422029:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202141.422032:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202141.422033:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202141.422036:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1272 00000100:00080000:1.0:1788202141.422038:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202141.422040:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202141.422048:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202141.422057:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559794048/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202141.422065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202141.422067:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202141.422069:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202141.422074:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1272 00000100:00080000:1.0:1788202141.422076:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202141.422077:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559793920:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202141.422079:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202141.422085:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202141.422092:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559794176/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202141.422156:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202141.422187:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202141/real 1788202141] req@ffff93c1c5037800 x1875064559793920/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202141.422202:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559793920/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202141.422210:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202141.422213:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202141.422216:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559793920:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202141.422235:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559794048:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202141.422250:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202141.422258:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202141/real 1788202141] req@ffff93c1c5036680 x1875064559794048/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202141.422266:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559794048/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202141.422272:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202141.422274:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202141.422276:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559794048:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202141.422280:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559794176:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202141.422291:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202141.422297:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202141/real 1788202141] req@ffff93c1c5035c00 x1875064559794176/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202141.422304:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559794176/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202141.422315:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202141.422316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202141.422318:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559794176:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202142.125967:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202143.149936:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202144.174005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202145.197943:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202146.030040:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202146.030053:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202146.030136:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202146.030177:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202146.030183:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202146.030216:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202146.222007:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202146.541944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202146.541957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202146.541961:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202146.541972:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1277 00000100:00080000:1.0:1788202146.541984:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202146.541992:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202146.542034:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559794304/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202146.542063:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202146.542066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202146.542068:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202146.542071:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1277 00000100:00080000:1.0:1788202146.542074:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202146.542078:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202146.542086:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202146.542094:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559794304:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788202146.542099:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559794432/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202146.542107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202146.542110:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202146.542111:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202146.542114:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1277 00000100:00080000:1.0:1788202146.542116:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202146.542118:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202146.542122:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202146.542131:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559794560/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202146.542157:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202146.542190:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202146/real 1788202146] req@ffff93c1c5035c00 x1875064559794304/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202146.542204:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559794304/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202146.542211:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202146.542214:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202146.542217:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559794304:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202146.542235:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559794432:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202146.542249:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202146.542257:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202146/real 1788202146] req@ffff93c1c5036680 x1875064559794432/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202146.542264:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559794432/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202146.542270:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202146.542272:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202146.542274:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559794432:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202146.542278:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559794560:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202146.542289:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202146.542295:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202146/real 1788202146] req@ffff93c1c5037800 x1875064559794560/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202146.542302:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559794560/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202146.542309:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202146.542311:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202146.542312:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559794560:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202147.246016:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202148.270026:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202149.293993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202150.317991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202151.149999:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202151.150020:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202151.150153:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202151.150175:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202151.150179:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202151.150201:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202151.342005:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202151.661987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202151.662005:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202151.662010:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202151.662021:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1282 00000100:00080000:2.0:1788202151.662032:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202151.662039:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202151.662091:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559794688/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202151.662126:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202151.662129:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202151.662131:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202151.662135:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1282 00000100:00080000:2.0:1788202151.662138:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202151.662142:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202151.662151:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202151.662164:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559794816/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202151.662173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202151.662176:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202151.662177:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202151.662185:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1282 00000100:00100000:3.0:1788202151.662186:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559794688:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202151.662190:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202151.662197:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202151.662203:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202151.662219:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559794944/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202151.662260:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202151.662323:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202151/real 1788202151] req@ffff93c0e62b8a80 x1875064559794688/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202151.662339:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559794688/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202151.662348:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202151.662351:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202151.662354:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559794688:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202151.662376:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559794816:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202151.662395:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202151.662403:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202151/real 1788202151] req@ffff93c0e62bad80 x1875064559794816/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202151.662413:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559794816/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202151.662419:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202151.662421:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202151.662423:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559794816:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202151.662427:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559794944:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202151.662439:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202151.662445:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202151/real 1788202151] req@ffff93c0e62bb100 x1875064559794944/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202151.662453:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559794944/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202151.662462:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202151.662465:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202151.662468:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559794944:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202152.366021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202153.389988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202154.413976:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202155.437987:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202156.270114:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202156.270131:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202156.270229:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202156.270252:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202156.270255:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202156.270277:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202156.461971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202156.781983:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202156.781997:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202156.782003:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202156.782016:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1287 00000100:00080000:0.0:1788202156.782031:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202156.782041:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202156.782090:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559795072/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202156.782116:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202156.782118:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202156.782120:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202156.782123:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1287 00000100:00080000:0.0:1788202156.782126:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202156.782129:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202156.782139:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202156.782150:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559795200/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202156.782158:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202156.782161:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202156.782163:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202156.782166:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1287 00000100:00080000:0.0:1788202156.782169:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202156.782174:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202156.782176:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559795072:192.168.201.156@tcp:250:kworker.0 00000008:00100000:0.0:1788202156.782178:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202156.782190:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559795328/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202156.782270:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202156.782320:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202156/real 1788202156] req@ffff93c0e9ce4380 x1875064559795072/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202156.782341:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559795072/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202156.782360:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202156.782364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202156.782369:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559795072:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202156.782377:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559795200:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202156.782400:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202156.782415:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202156/real 1788202156] req@ffff93c0e9ce4700 x1875064559795200/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202156.782428:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559795200/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202156.782436:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202156.782440:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202156.782443:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559795200:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202156.782475:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559795328:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202156.782496:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202156.782507:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202156/real 1788202156] req@ffff93c0e9ce7480 x1875064559795328/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202156.782529:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559795328/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202156.782542:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202156.782545:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202156.782549:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559795328:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202157.485973:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202158.510001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202159.533984:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202160.557939:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202161.390073:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202161.390090:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202161.390234:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202161.390260:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202161.390263:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202161.390283:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202161.582007:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202161.901921:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202161.901934:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202161.901940:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202161.901949:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1292 00000100:00080000:0.0:1788202161.901962:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202161.901972:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202161.902006:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559795456/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202161.902037:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202161.902039:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202161.902041:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202161.902044:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1292 00000100:00080000:0.0:1788202161.902046:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202161.902049:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202161.902057:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202161.902066:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559795584/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202161.902073:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202161.902075:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202161.902076:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202161.902079:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1292 00000100:00080000:0.0:1788202161.902082:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202161.902087:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202161.902091:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202161.902098:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559795712/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202161.902132:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559795456:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788202161.902209:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202161.902263:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202161/real 1788202161] req@ffff93c0e9ce7480 x1875064559795456/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202161.902279:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559795456/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202161.902289:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202161.902292:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202161.902296:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559795456:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202161.902302:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559795584:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202161.902318:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202161.902326:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202161/real 1788202161] req@ffff93c0e9ce4700 x1875064559795584/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202161.902334:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559795584/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202161.902340:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202161.902342:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202161.902344:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559795584:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202161.902368:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559795712:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202161.902380:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202161.902386:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202161/real 1788202161] req@ffff93c0e9ce4380 x1875064559795712/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202161.902395:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559795712/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202161.902403:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202161.902404:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202161.902406:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559795712:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202162.605964:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202163.629982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202164.653932:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202165.677983:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202166.509979:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202166.509996:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202166.510226:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202166.510249:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202166.510253:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202166.510281:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202166.702019:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202167.021875:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202167.021885:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202167.021889:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202167.021900:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1297 00000100:00080000:0.0:1788202167.021913:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202167.021921:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202167.021969:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559795840/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202167.022008:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202167.022012:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202167.022014:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202167.022032:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1297 00000100:00080000:0.0:1788202167.022038:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202167.022043:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202167.022056:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202167.022062:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559795840:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202167.022077:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559795968/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202167.022092:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202167.022096:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202167.022099:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202167.022103:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1297 00000100:00080000:0.0:1788202167.022108:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202167.022111:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202167.022116:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202167.022126:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559796096/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202167.022135:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202167.022175:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202167/real 1788202167] req@ffff93c0e9ce6a00 x1875064559795840/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202167.022191:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559795840/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202167.022209:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202167.022214:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202167.022219:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559795840:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202167.022256:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559795968:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202167.022279:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202167.022293:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202167/real 1788202167] req@ffff93c0e9ce4000 x1875064559795968/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202167.022306:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559795968/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202167.022324:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202167.022328:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202167.022332:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559795968:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202167.022338:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559796096:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202167.022356:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202167.022365:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202167/real 1788202167] req@ffff93c0e9ce6680 x1875064559796096/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202167.022378:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559796096/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202167.022390:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202167.022394:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202167.022397:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559796096:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202167.725993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202168.750033:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202169.774067:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202170.798040:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202171.629993:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202171.630008:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202171.630090:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202171.630111:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202171.630114:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202171.630142:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202171.822033:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202172.141966:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202172.141976:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202172.141980:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202172.141991:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1303 00000100:00080000:0.0:1788202172.142003:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202172.142010:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202172.142053:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559796224/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202172.142086:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202172.142089:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202172.142091:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202172.142096:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1303 00000100:00080000:0.0:1788202172.142100:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202172.142106:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202172.142119:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202172.142136:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559796224:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202172.142137:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559796352/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202172.142153:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202172.142156:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202172.142157:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202172.142161:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1303 00000100:00080000:0.0:1788202172.142165:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202172.142168:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202172.142173:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202172.142184:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559796480/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202172.142252:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559796352:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202172.142272:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559796480:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202172.846076:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202173.870037:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202174.893965:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788202175.214231:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788202175.214282:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788202175.214303:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202175.214358:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202175.214370:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788202175.214431:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202172/real 1788202175] req@ffff93c0e9ce5c00 x1875064559796352/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202175.214464:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559796352/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202175.214488:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202175.214495:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202175.214501:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559796352:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202175.214512:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202172/real 1788202175] req@ffff93c0e9ce6d80 x1875064559796480/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202175.214526:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559796480/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202175.214540:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202175.214544:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202175.214548:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559796480:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788202175.214557:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202172/real 1788202175] req@ffff93c0e9ce7b80 x1875064559796224/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202175.214569:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559796224/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202175.214577:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202175.214581:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202175.214584:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559796224:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788202175.918008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202176.750109:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202176.750129:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202176.750277:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202176.750317:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202176.750323:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202176.750360:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202176.942021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202177.261964:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202177.261979:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202177.261984:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202177.261997:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1308 00000100:00080000:0.0:1788202177.262011:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202177.262021:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202177.262069:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559796608/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202177.262105:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202177.262109:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202177.262112:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202177.262117:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1308 00000100:00080000:0.0:1788202177.262121:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202177.262126:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202177.262140:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202177.262154:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559796736/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202177.262164:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788202177.262164:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559796608:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202177.262172:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202177.262175:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202177.262180:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1308 00000100:00080000:0.0:1788202177.262182:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202177.262185:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202177.262189:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202177.262199:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559796864/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202177.262240:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202177.262281:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202177/real 1788202177] req@ffff93c0e9ce7100 x1875064559796608/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202177.262303:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559796608/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202177.262315:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202177.262321:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202177.262326:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559796608:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202177.262358:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559796736:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202177.262383:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202177.262395:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202177/real 1788202177] req@ffff93c0e9ce5f80 x1875064559796736/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202177.262405:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559796736/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202177.262412:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202177.262414:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202177.262417:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559796736:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202177.262421:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559796864:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202177.262435:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202177.262442:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202177/real 1788202177] req@ffff93c0e9ce4a80 x1875064559796864/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202177.262450:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559796864/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202177.262462:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202177.262465:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202177.262469:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559796864:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202177.966002:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202178.989999:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202180.014002:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202181.037991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202181.870116:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202181.870132:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202181.870226:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202181.870252:0:18311:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202181.870257:0:18311:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202181.870299:0:18311:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202182.061922:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202182.381938:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202182.381947:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202182.381951:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202182.381960:0:18311:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1313 00000100:00080000:0.0:1788202182.381971:0:18311:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202182.381977:0:18311:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202182.382024:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559796992/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202182.382056:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202182.382059:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202182.382061:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202182.382066:0:18311:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1313 00000100:00080000:0.0:1788202182.382070:0:18311:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202182.382074:0:18311:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202182.382087:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202182.382101:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559797120/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202182.382105:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559796992:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202182.382113:0:18311:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202182.382116:0:18311:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202182.382118:0:18311:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202182.382121:0:18311:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1313 00000100:00080000:0.0:1788202182.382123:0:18311:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202182.382126:0:18311:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202182.382130:0:18311:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202182.382137:0:18311:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559797248/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202182.382170:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202182.382214:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202182/real 1788202182] req@ffff93c0e9ce5500 x1875064559796992/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202182.382233:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559796992/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202182.382242:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202182.382245:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202182.382248:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559796992:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202182.382271:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559797120:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202182.382292:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202182.382305:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202182/real 1788202182] req@ffff93c0e9ce7800 x1875064559797120/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202182.382317:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559797120/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202182.382325:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202182.382328:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202182.382330:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559797120:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202182.382334:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559797248:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202182.382349:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202182.382356:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202182/real 1788202182] req@ffff93c0e9ce5180 x1875064559797248/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202182.382364:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559797248/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202182.382370:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202182.382372:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202182.382374:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559797248:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202183.085998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202184.109917:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202185.134006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202186.157979:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202186.990038:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202186.990053:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202186.990154:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202186.990172:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202186.990174:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202186.990189:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202187.181970:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202187.501987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202187.501995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202187.501999:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202187.502007:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1318 00000100:00080000:2.0:1788202187.502016:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202187.502023:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202187.502061:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559797376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202187.502083:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202187.502086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202187.502087:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202187.502089:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1318 00000100:00080000:2.0:1788202187.502092:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202187.502095:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202187.502102:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202187.502110:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559797504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202187.502121:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202187.502123:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202187.502125:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202187.502128:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1318 00000100:00080000:2.0:1788202187.502131:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202187.502133:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202187.502137:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202187.502144:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559797632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202187.502173:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559797376:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788202187.502254:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202187.502304:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202187/real 1788202187] req@ffff93c0e62b9c00 x1875064559797376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202187.502323:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559797376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202187.502335:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202187.502340:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202187.502344:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559797376:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202187.502352:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559797504:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202187.502374:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202187.502388:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202187/real 1788202187] req@ffff93c0e62bbb80 x1875064559797504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202187.502415:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559797504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202187.502424:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202187.502427:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202187.502430:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559797504:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202187.502437:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559797632:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202187.502458:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202187.502469:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202187/real 1788202187] req@ffff93c0e62b8380 x1875064559797632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202187.502481:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559797632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202187.502491:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202187.502495:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202187.502499:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559797632:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202188.205992:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202189.230003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202190.254019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202191.277985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202192.109999:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202192.110015:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202192.110118:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202192.110134:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202192.110137:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202192.110152:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202192.301980:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202192.621925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202192.621936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202192.621939:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202192.621949:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1323 00000100:00080000:2.0:1788202192.621959:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202192.621966:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202192.622007:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559797760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202192.622043:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202192.622047:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202192.622049:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202192.622054:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1323 00000100:00080000:2.0:1788202192.622058:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202192.622062:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202192.622072:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202192.622083:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559797888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202192.622088:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559797760:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202192.622095:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202192.622099:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202192.622101:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202192.622105:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1323 00000100:00080000:2.0:1788202192.622107:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202192.622109:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202192.622113:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202192.622120:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559798016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202192.622156:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202192.622201:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202192/real 1788202192] req@ffff93c0e62ba680 x1875064559797760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202192.622215:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559797760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202192.622227:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202192.622230:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202192.622233:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559797760:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202192.622252:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559797888:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202192.622267:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202192.622277:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202192/real 1788202192] req@ffff93c0e62b8000 x1875064559797888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202192.622288:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559797888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202192.622296:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202192.622299:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202192.622303:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559797888:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202192.622309:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559798016:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202192.622327:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202192.622335:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202192/real 1788202192] req@ffff93c0e62b8700 x1875064559798016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202192.622342:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559798016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202192.622349:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202192.622351:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202192.622353:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559798016:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202193.326021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202194.350002:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202195.373970:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202196.397978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202197.229996:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202197.230011:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202197.230107:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202197.230125:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202197.230127:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202197.230142:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202197.421947:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202197.741937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202197.741947:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202197.741951:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202197.741960:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1328 00000100:00080000:2.0:1788202197.741978:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202197.741985:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202197.742024:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559798144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202197.742047:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202197.742050:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202197.742051:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202197.742054:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1328 00000100:00080000:2.0:1788202197.742056:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202197.742059:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202197.742067:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202197.742076:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559798272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202197.742083:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202197.742086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202197.742087:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202197.742090:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1328 00000100:00080000:2.0:1788202197.742092:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202197.742094:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202197.742098:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202197.742101:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559798144:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202197.742108:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559798400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202197.742167:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202197.742208:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202197/real 1788202197] req@ffff93c0e62baa00 x1875064559798144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202197.742222:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559798144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202197.742230:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202197.742232:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202197.742235:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559798144:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202197.742240:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559798272:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202197.742253:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202197.742260:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202197/real 1788202197] req@ffff93c0e62bb480 x1875064559798272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202197.742267:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559798272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202197.742272:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202197.742273:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202197.742275:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559798272:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202197.742295:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559798400:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202197.742305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202197.742311:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202197/real 1788202197] req@ffff93c0e62b8700 x1875064559798400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202197.742318:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559798400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202197.742324:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202197.742326:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202197.742327:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559798400:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202198.446004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202199.469928:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202200.493996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202201.517953:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202202.349999:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202202.350011:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202202.350094:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202202.350111:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202202.350113:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202202.350128:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202202.541986:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202202.861994:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202202.862003:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202202.862007:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202202.862017:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1333 00000100:00080000:2.0:1788202202.862028:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202202.862035:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202202.862074:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559798528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202202.862095:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202202.862100:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202202.862102:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202202.862106:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1333 00000100:00080000:2.0:1788202202.862108:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202202.862111:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202202.862118:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202202.862130:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559798656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202202.862130:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559798528:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202202.862154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202202.862156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202202.862157:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202202.862160:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1333 00000100:00080000:2.0:1788202202.862166:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202202.862169:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202202.862171:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202202.862179:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559798784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202202.862212:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202202.862248:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202202/real 1788202202] req@ffff93c0e62b8000 x1875064559798528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202202.862264:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559798528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202202.862273:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202202.862277:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202202.862281:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559798528:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202202.862302:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559798656:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202202.862323:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202202.862336:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202202/real 1788202202] req@ffff93c0e62ba680 x1875064559798656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202202.862350:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559798656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202202.862367:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202202.862370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202202.862372:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559798656:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202202.862377:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559798784:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202202.862395:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202202.862417:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202202/real 1788202202] req@ffff93c0e62b8380 x1875064559798784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202202.862426:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559798784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202202.862434:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202202.862436:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202202.862438:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559798784:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202203.565994:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202204.589975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202205.613971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202206.637972:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202207.470000:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202207.470017:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202207.470110:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202207.470134:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202207.470138:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202207.470164:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202207.662004:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202207.981968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202207.981979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202207.981983:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202207.981992:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1338 00000100:00080000:2.0:1788202207.982003:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202207.982010:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202207.982049:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559798912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202207.982076:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202207.982080:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202207.982081:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202207.982085:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1338 00000100:00080000:2.0:1788202207.982089:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202207.982093:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202207.982102:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202207.982116:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559799040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202207.982130:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559798912:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202207.982133:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202207.982147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202207.982149:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202207.982154:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1338 00000100:00080000:2.0:1788202207.982157:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202207.982161:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202207.982166:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202207.982179:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559799168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202207.982213:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202207.982263:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202207/real 1788202207] req@ffff93c0e62bbb80 x1875064559798912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202207.982283:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559798912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202207.982297:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202207.982302:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202207.982307:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559798912:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202207.982352:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559799040:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202207.982373:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202207.982383:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202207/real 1788202207] req@ffff93c0e62b9c00 x1875064559799040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202207.982392:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559799040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202207.982399:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202207.982401:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202207.982404:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559799040:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202207.982408:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559799168:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202207.982426:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202207.982433:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202207/real 1788202207] req@ffff93c0e62bb100 x1875064559799168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202207.982441:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559799168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202207.982449:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202207.982451:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202207.982454:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559799168:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202208.685993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202209.709981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202210.733944:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202211.758040:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202212.590094:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202212.590110:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202212.590225:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202212.590286:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202212.590290:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202212.590329:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202212.782065:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202213.101958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202213.101969:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202213.101972:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202213.101991:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1343 00000100:00080000:3.0:1788202213.102003:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202213.102013:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202213.102054:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559799296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202213.102079:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202213.102082:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202213.102084:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202213.102088:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1343 00000100:00080000:3.0:1788202213.102091:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202213.102094:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202213.102103:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202213.102116:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559799296:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202213.102120:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559799424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202213.102130:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202213.102133:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202213.102134:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202213.102138:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1343 00000100:00080000:3.0:1788202213.102141:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202213.102144:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202213.102148:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202213.102158:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559799552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202213.102185:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202213.102243:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202213/real 1788202213] req@ffff93c1c4a4d180 x1875064559799296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202213.102273:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559799296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202213.102283:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202213.102286:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202213.102289:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559799296:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202213.102312:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559799424:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202213.102332:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202213.102341:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202213/real 1788202213] req@ffff93c1c4a4d880 x1875064559799424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202213.102350:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559799424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202213.102356:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202213.102358:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202213.102360:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559799424:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202213.102364:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559799552:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202213.102376:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202213.102383:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202213/real 1788202213] req@ffff93c1c4a4e300 x1875064559799552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202213.102391:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559799552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202213.102402:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202213.102404:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202213.102407:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559799552:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202213.806010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202214.830034:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202215.853989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202216.877990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202217.710066:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202217.710084:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202217.710233:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202217.710266:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202217.710272:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202217.710302:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202217.901983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202218.221920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202218.221930:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202218.221934:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202218.221944:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1349 00000100:00080000:1.0:1788202218.221955:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202218.221962:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202218.222001:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559799680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202218.222029:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202218.222038:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202218.222040:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202218.222046:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1349 00000100:00080000:1.0:1788202218.222050:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202218.222056:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202218.222068:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202218.222087:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559799808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788202218.222088:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559799680:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202218.222108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202218.222113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202218.222116:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202218.222122:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1349 00000100:00080000:1.0:1788202218.222126:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202218.222157:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202218.222162:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202218.222175:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559799936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202218.222190:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202218.222241:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202218/real 1788202218] req@ffff93c1c5037100 x1875064559799680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202218.222257:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559799680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202218.222266:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202218.222269:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202218.222272:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559799680:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202218.222295:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559799808:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202218.222312:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202218.222320:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202218/real 1788202218] req@ffff93c1c5034e00 x1875064559799808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202218.222328:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559799808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202218.222334:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202218.222336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202218.222338:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559799808:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202218.222343:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559799936:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202218.222354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202218.222360:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202218/real 1788202218] req@ffff93c1c5034a80 x1875064559799936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202218.222368:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559799936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202218.222375:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202218.222377:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202218.222379:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559799936:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202218.926007:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202219.949971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202220.973951:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202221.998766:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202222.830011:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202222.830028:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202222.830187:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202222.830216:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202222.830219:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202222.830242:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202223.021906:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202223.341970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202223.341980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202223.341983:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202223.341994:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1354 00000100:00080000:1.0:1788202223.342005:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202223.342012:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202223.342080:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559800064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202223.342106:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202223.342109:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202223.342110:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202223.342113:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1354 00000100:00080000:1.0:1788202223.342117:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202223.342119:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202223.342129:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202223.342140:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559800192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202223.342148:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202223.342151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202223.342154:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788202223.342156:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559800064:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202223.342160:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1354 00000100:00080000:1.0:1788202223.342167:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202223.342170:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202223.342174:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202223.342181:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559800320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202223.342236:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202223.342276:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202223/real 1788202223] req@ffff93c1c5034a80 x1875064559800064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202223.342291:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559800064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202223.342300:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202223.342312:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202223.342315:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559800064:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202223.342339:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559800192:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202223.342361:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202223.342371:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202223/real 1788202223] req@ffff93c1c5034e00 x1875064559800192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202223.342382:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559800192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202223.342392:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202223.342395:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202223.342399:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559800192:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202223.342406:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559800320:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202223.342428:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202223.342440:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202223/real 1788202223] req@ffff93c1c5037100 x1875064559800320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202223.342454:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559800320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202223.342466:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202223.342470:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202223.342473:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559800320:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202224.045970:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202225.070080:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202226.094011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202227.118517:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202227.950039:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202227.950055:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202227.950172:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202227.950195:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202227.950199:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202227.950220:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202228.142013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202228.462013:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202228.462021:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202228.462025:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202228.462033:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1359 00000100:00080000:0.0:1788202228.462042:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202228.462048:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202228.462083:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559800448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202228.462106:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202228.462109:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202228.462110:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202228.462113:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1359 00000100:00080000:0.0:1788202228.462114:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202228.462116:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202228.462123:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202228.462130:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559800576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202228.462137:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202228.462139:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202228.462141:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202228.462144:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1359 00000100:00080000:0.0:1788202228.462148:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202228.462150:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202228.462154:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202228.462161:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559800704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788202228.462197:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559800448:192.168.201.156@tcp:250:kworker.0 00000800:00000100:2.0:1788202228.462287:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202228.462343:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202228/real 1788202228] req@ffff93c0e9ce4e00 x1875064559800448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202228.462364:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559800448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202228.462379:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202228.462382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202228.462386:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559800448:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202228.462392:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559800576:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202228.462412:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202228.462425:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202228/real 1788202228] req@ffff93c0e9ce5880 x1875064559800576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202228.462439:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559800576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202228.462449:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202228.462453:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202228.462456:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559800576:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202228.462463:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559800704:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202228.462481:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202228.462489:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202228/real 1788202228] req@ffff93c0e9ce6300 x1875064559800704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202228.462498:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559800704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202228.462506:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202228.462508:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202228.462510:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559800704:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202229.166048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202230.190000:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202231.213987:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202232.237977:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202233.069995:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202233.070008:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202233.070087:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202233.070110:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202233.070113:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202233.070129:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202233.262000:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202233.581946:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202233.581954:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202233.581958:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202233.581966:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1364 00000100:00080000:0.0:1788202233.581975:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202233.581981:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202233.582013:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559800832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202233.582034:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202233.582037:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202233.582038:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202233.582041:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1364 00000100:00080000:0.0:1788202233.582043:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202233.582051:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202233.582058:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202233.582066:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559800960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202233.582076:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202233.582079:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202233.582081:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788202233.582085:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559800832:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202233.582087:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1364 00000100:00080000:0.0:1788202233.582093:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202233.582097:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202233.582103:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202233.582117:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559801088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202233.582162:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202233.582199:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202233/real 1788202233] req@ffff93c0e9ce6300 x1875064559800832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202233.582216:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559800832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202233.582227:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202233.582230:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202233.582234:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559800832:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202233.582258:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559800960:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202233.582290:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202233.582303:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202233/real 1788202233] req@ffff93c0e9ce5880 x1875064559800960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202233.582315:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559800960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202233.582323:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202233.582326:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202233.582328:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559800960:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202233.582334:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559801088:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202233.582352:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202233.582362:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202233/real 1788202233] req@ffff93c0e9ce4e00 x1875064559801088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202233.582373:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559801088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202233.582383:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202233.582386:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202233.582388:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559801088:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202234.286002:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202235.309999:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202236.333930:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202237.357994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202238.189963:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202238.189976:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202238.190059:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202238.190080:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202238.190084:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202238.190113:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202238.382212:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202238.701958:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202238.701969:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202238.701974:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202238.701985:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1369 00000100:00080000:0.0:1788202238.701996:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202238.702003:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202238.702044:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559801216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202238.702073:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202238.702076:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202238.702078:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202238.702083:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1369 00000100:00080000:0.0:1788202238.702086:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202238.702089:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202238.702099:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202238.702111:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559801344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202238.702121:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202238.702124:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202238.702126:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202238.702131:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1369 00000100:00080000:0.0:1788202238.702137:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202238.702140:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202238.702145:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202238.702155:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559801472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788202238.702165:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559801216:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202238.702285:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559801344:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202238.702331:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559801472:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202239.405997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202240.430346:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202241.453975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788202241.774217:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788202241.774273:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788202241.774297:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788202241.774341:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788202241.774350:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:2.0:1788202241.774415:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202238/real 1788202241] req@ffff93c0e9ce5500 x1875064559801472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202241.774444:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559801472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202241.774459:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202241.774464:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202241.774468:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559801472:192.168.201.156@tcp:8:kworker.0 00000100:00100000:2.0:1788202241.774488:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202238/real 1788202241] req@ffff93c0e9ce5180 x1875064559801216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202241.774500:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559801216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202241.774513:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202241.774516:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202241.774519:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559801216:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202241.774533:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202238/real 1788202241] req@ffff93c0e9ce7800 x1875064559801344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202241.774545:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559801344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202241.774552:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202241.774555:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202241.774558:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559801344:192.168.201.156@tcp:38:kworker.0 00010000:00010000:0.0:1788202242.477975:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202243.310024:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202243.310041:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202243.310142:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202243.310167:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202243.310172:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202243.310200:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202243.501976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202243.822008:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202243.822019:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202243.822023:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202243.822034:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1374 00000100:00080000:2.0:1788202243.822047:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202243.822055:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202243.822095:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559801600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202243.822127:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202243.822132:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202243.822135:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202243.822143:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1374 00000100:00080000:2.0:1788202243.822148:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202243.822153:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202243.822165:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202243.822181:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559801728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202243.822190:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559801600:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202243.822198:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202243.822204:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202243.822206:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202243.822211:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1374 00000100:00080000:2.0:1788202243.822214:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202243.822219:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202243.822224:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202243.822236:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559801856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202243.822292:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202243.822347:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202243/real 1788202243] req@ffff93c0e62bad80 x1875064559801600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202243.822369:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559801600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202243.822381:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202243.822386:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202243.822391:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559801600:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202243.822425:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559801728:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202243.822449:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202243.822462:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202243/real 1788202243] req@ffff93c0e62b8a80 x1875064559801728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202243.822482:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559801728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202243.822490:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202243.822493:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202243.822497:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559801728:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202243.822503:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559801856:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202243.822522:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202243.822534:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202243/real 1788202243] req@ffff93c0e62b9500 x1875064559801856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202243.822546:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559801856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202243.822556:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202243.822559:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202243.822563:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559801856:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202244.526014:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202245.549982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202246.573985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202247.597988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202248.430015:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202248.430030:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202248.430127:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202248.430146:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202248.430150:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202248.430179:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202248.621929:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202248.941952:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202248.941963:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202248.941967:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202248.941977:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1379 00000100:00080000:2.0:1788202248.941988:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202248.942002:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202248.942033:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559801984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202248.942055:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202248.942058:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202248.942059:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202248.942061:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1379 00000100:00080000:2.0:1788202248.942064:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202248.942066:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202248.942073:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202248.942081:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559802112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202248.942088:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202248.942090:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202248.942092:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202248.942097:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1379 00000100:00080000:2.0:1788202248.942099:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202248.942104:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202248.942106:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559801984:192.168.201.156@tcp:250:kworker.0 00000008:00100000:2.0:1788202248.942114:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202248.942129:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559802240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202248.942168:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202248.942198:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202248/real 1788202248] req@ffff93c0e62bb800 x1875064559801984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202248.942211:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559801984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202248.942218:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202248.942221:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202248.942224:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559801984:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202248.942229:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559802112:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202248.942241:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202248.942249:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202248/real 1788202248] req@ffff93c0e62b9f80 x1875064559802112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202248.942256:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559802112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202248.942261:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202248.942262:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202248.942264:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559802112:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202248.942281:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559802240:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202248.942291:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202248.942296:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202248/real 1788202248] req@ffff93c0e62b9500 x1875064559802240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202248.942303:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559802240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202248.942313:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202248.942315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202248.942317:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559802240:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202249.645953:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202250.669975:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202251.693982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202252.717972:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202253.550027:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202253.550045:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202253.550188:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202253.550217:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202253.550221:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202253.550252:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202253.741968:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202254.061954:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202254.061963:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202254.061966:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202254.061977:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1384 00000100:00080000:2.0:1788202254.061988:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202254.061995:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202254.062035:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559802368/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202254.062060:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202254.062062:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202254.062063:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202254.062067:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1384 00000100:00080000:2.0:1788202254.062070:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202254.062072:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202254.062081:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202254.062091:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559802496/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202254.062098:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202254.062100:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202254.062101:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202254.062103:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1384 00000100:00080000:2.0:1788202254.062106:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202254.062108:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202254.062111:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202254.062117:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559802368:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202254.062119:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559802624/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202254.062197:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202254.062248:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202254/real 1788202254] req@ffff93c0e62b8a80 x1875064559802368/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202254.062267:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559802368/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202254.062288:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202254.062296:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202254.062299:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559802368:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202254.062307:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559802496:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202254.062330:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202254.062342:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202254/real 1788202254] req@ffff93c0e62bad80 x1875064559802496/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202254.062353:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559802496/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202254.062360:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202254.062362:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202254.062364:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559802496:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202254.062387:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559802624:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202254.062407:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202254.062418:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202254/real 1788202254] req@ffff93c0e62bb100 x1875064559802624/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202254.062429:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559802624/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202254.062440:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202254.062443:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202254.062446:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559802624:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202254.765992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202255.789982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202256.814001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202257.837954:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202258.670012:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202258.670029:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202258.670142:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202258.670160:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202258.670162:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202258.670178:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202258.862137:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202259.181938:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202259.181947:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202259.181950:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202259.181960:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1390 00000100:00080000:2.0:1788202259.181971:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202259.181977:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202259.182021:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559802752/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202259.182053:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202259.182056:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202259.182057:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202259.182061:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1390 00000100:00080000:2.0:1788202259.182063:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202259.182066:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202259.182076:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202259.182087:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559802880/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202259.182094:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202259.182097:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202259.182099:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202259.182102:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1390 00000100:00080000:2.0:1788202259.182105:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202259.182107:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202259.182110:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202259.182119:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559803008/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202259.182139:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559802752:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202259.182224:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202259.182265:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202259/real 1788202259] req@ffff93c0e62b9c00 x1875064559802752/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202259.182281:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559802752/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202259.182290:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202259.182293:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202259.182297:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559802752:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202259.182302:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559802880:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202259.182319:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202259.182336:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202259/real 1788202259] req@ffff93c0e62bbb80 x1875064559802880/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202259.182345:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559802880/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202259.182351:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202259.182354:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202259.182356:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559802880:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202259.182378:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559803008:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202259.182391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202259.182404:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202259/real 1788202259] req@ffff93c0e62b8380 x1875064559803008/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202259.182415:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559803008/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202259.182436:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202259.182439:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202259.182441:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559803008:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202259.885949:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202260.910019:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202261.934007:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202262.957989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202263.790013:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202263.790034:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202263.790158:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202263.790183:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202263.790186:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202263.790205:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788202263.982032:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202264.302057:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202264.302067:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202264.302071:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202264.302080:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1395 00000100:00080000:1.0:1788202264.302091:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202264.302097:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202264.302133:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559803136/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202264.302153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202264.302156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202264.302157:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202264.302160:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1395 00000100:00080000:1.0:1788202264.302162:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202264.302165:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202264.302173:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202264.302181:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559803264/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202264.302189:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202264.302192:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202264.302193:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788202264.302197:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559803136:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202264.302200:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1395 00000100:00080000:1.0:1788202264.302206:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202264.302209:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202264.302213:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202264.302222:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559803392/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202264.302263:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202264.302296:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202264/real 1788202264] req@ffff93c1c5037800 x1875064559803136/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202264.302311:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559803136/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202264.302319:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202264.302322:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202264.302353:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559803136:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202264.302379:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559803264:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202264.302397:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202264.302406:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202264/real 1788202264] req@ffff93c1c5036680 x1875064559803264/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202264.302415:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559803264/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202264.302421:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202264.302423:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202264.302425:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559803264:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202264.302429:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559803392:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202264.302441:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202264.302447:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202264/real 1788202264] req@ffff93c1c5035c00 x1875064559803392/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202264.302454:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559803392/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202264.302462:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202264.302464:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202264.302466:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559803392:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202265.006023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202266.030009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202267.053985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202268.077983:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202268.910024:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202268.910042:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202268.910151:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202268.910179:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202268.910184:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202268.910217:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202269.102010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202269.421963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202269.421974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202269.421978:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202269.421989:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1400 00000100:00080000:1.0:1788202269.422001:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202269.422008:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202269.422052:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559803520/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202269.422078:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202269.422081:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202269.422083:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202269.422086:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1400 00000100:00080000:1.0:1788202269.422088:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202269.422092:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202269.422102:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202269.422113:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559803648/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202269.422129:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559803520:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202269.422129:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202269.422137:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202269.422139:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202269.422144:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1400 00000100:00080000:1.0:1788202269.422147:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202269.422149:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202269.422153:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202269.422162:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559803776/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202269.422202:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202269.422261:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202269/real 1788202269] req@ffff93c1c5035c00 x1875064559803520/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202269.422277:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559803520/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202269.422285:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202269.422289:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202269.422292:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559803520:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202269.422314:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559803648:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202269.422332:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202269.422341:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202269/real 1788202269] req@ffff93c1c5036680 x1875064559803648/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202269.422349:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559803648/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202269.422355:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202269.422358:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202269.422361:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559803648:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202269.422365:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559803776:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202269.422376:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202269.422382:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202269/real 1788202269] req@ffff93c1c5037800 x1875064559803776/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202269.422389:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559803776/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202269.422409:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202269.422411:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202269.422413:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559803776:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788202270.126063:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202271.150001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202272.174060:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202273.198006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202274.030011:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202274.030032:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202274.030175:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202274.030209:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202274.030215:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202274.030252:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788202274.222027:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202274.541908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202274.541918:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202274.541922:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202274.541932:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1405 00000100:00080000:1.0:1788202274.541943:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202274.541950:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202274.542003:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559803904/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202274.542025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202274.542028:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202274.542032:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202274.542036:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1405 00000100:00080000:1.0:1788202274.542038:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202274.542042:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202274.542052:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202274.542062:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559804032/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202274.542070:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202274.542075:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788202274.542076:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559803904:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202274.542077:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202274.542085:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1405 00000100:00080000:1.0:1788202274.542088:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202274.542090:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202274.542094:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202274.542102:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559804160/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202274.542180:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202274.542234:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202274/real 1788202274] req@ffff93c1c5037100 x1875064559803904/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202274.542257:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559803904/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202274.542266:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202274.542269:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202274.542272:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559803904:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202274.542313:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559804032:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202274.542341:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202274.542354:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202274/real 1788202274] req@ffff93c1c5034e00 x1875064559804032/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202274.542364:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559804032/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202274.542370:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202274.542372:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202274.542374:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559804032:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202274.542379:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559804160:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202274.542404:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202274.542412:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202274/real 1788202274] req@ffff93c1c5034a80 x1875064559804160/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202274.542420:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559804160/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202274.542437:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202274.542441:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202274.542444:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559804160:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202275.245929:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202276.270075:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202277.293966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202278.318040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202279.150028:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202279.150046:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202279.150171:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202279.150198:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202279.150203:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202279.150236:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202279.341915:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202279.661953:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202279.661962:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202279.661966:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202279.661974:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1410 00000100:00080000:2.0:1788202279.661984:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202279.661990:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202279.662022:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559804288/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202279.662046:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202279.662049:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202279.662050:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202279.662053:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1410 00000100:00080000:2.0:1788202279.662056:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202279.662058:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202279.662091:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202279.662103:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559804416/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202279.662112:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202279.662114:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202279.662116:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202279.662121:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1410 00000100:00100000:0.0:1788202279.662124:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559804288:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202279.662125:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202279.662132:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202279.662136:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202279.662147:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559804544/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202279.662189:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202279.662223:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202279/real 1788202279] req@ffff93c0e62ba680 x1875064559804288/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202279.662238:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559804288/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202279.662246:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202279.662249:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202279.662252:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559804288:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202279.662257:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559804416:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202279.662271:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202279.662283:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202279/real 1788202279] req@ffff93c0e62b8000 x1875064559804416/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202279.662296:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559804416/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202279.662310:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202279.662314:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202279.662317:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559804416:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202279.662343:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559804544:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202279.662364:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202279.662376:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202279/real 1788202279] req@ffff93c0e62b8700 x1875064559804544/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202279.662389:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559804544/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202279.662401:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202279.662404:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202279.662408:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559804544:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788202280.366043:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202281.389986:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202282.414081:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202283.437995:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202284.270028:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202284.270046:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202284.270147:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202284.270180:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202284.270187:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202284.270221:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788202284.462026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202284.782001:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202284.782011:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202284.782016:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202284.782027:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1415 00000100:00080000:1.0:1788202284.782040:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202284.782047:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202284.782093:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559804672/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202284.782117:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202284.782120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202284.782122:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202284.782125:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1415 00000100:00080000:1.0:1788202284.782128:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202284.782130:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202284.782140:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202284.782150:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559804800/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202284.782160:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202284.782164:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202284.782165:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202284.782168:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1415 00000100:00080000:1.0:1788202284.782170:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202284.782172:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202284.782176:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202284.782184:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559804928/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202284.782198:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559804672:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202284.782285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202284.782346:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202284/real 1788202284] req@ffff93c1c5034a80 x1875064559804672/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202284.782368:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559804672/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202284.782377:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202284.782382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202284.782386:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559804672:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202284.782393:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559804800:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202284.782425:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202284.782436:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202284/real 1788202284] req@ffff93c1c5034e00 x1875064559804800/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202284.782444:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559804800/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202284.782450:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202284.782452:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202284.782455:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559804800:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202284.782474:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559804928:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202284.782487:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202284.782493:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202284/real 1788202284] req@ffff93c1c5037100 x1875064559804928/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202284.782501:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559804928/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202284.782508:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202284.782510:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202284.782512:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559804928:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202285.486046:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202286.510053:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202287.533982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202288.558008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202289.390013:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202289.390027:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202289.390116:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202289.390135:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202289.390138:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202289.390158:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202289.581997:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202289.901970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202289.901980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202289.901983:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202289.901994:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1420 00000100:00080000:2.0:1788202289.902007:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202289.902013:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202289.902056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559805056/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202289.902085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202289.902088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202289.902089:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202289.902093:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1420 00000100:00080000:2.0:1788202289.902095:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202289.902098:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202289.902108:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202289.902118:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559805184/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202289.902127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202289.902130:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202289.902131:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202289.902136:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1420 00000100:00100000:0.0:1788202289.902139:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559805056:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202289.902139:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202289.902146:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202289.902152:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202289.902162:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559805312/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202289.902223:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202289.902268:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202289/real 1788202289] req@ffff93c0e62bb480 x1875064559805056/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202289.902285:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559805056/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202289.902297:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202289.902302:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202289.902307:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559805056:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202289.902345:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559805184:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202289.902366:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202289.902376:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202289/real 1788202289] req@ffff93c0e62baa00 x1875064559805184/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202289.902385:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559805184/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202289.902391:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202289.902394:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202289.902397:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559805184:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202289.902401:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559805312:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202289.902414:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202289.902421:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202289/real 1788202289] req@ffff93c0e62b8700 x1875064559805312/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202289.902429:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559805312/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202289.902436:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202289.902438:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202289.902440:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559805312:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788202290.606041:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202291.629992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202292.654022:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202293.678030:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202294.510031:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202294.510052:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202294.510186:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202294.510212:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202294.510216:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202294.510237:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788202294.702009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202295.021931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202295.021945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202295.021950:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202295.021963:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1425 00000100:00080000:1.0:1788202295.021976:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202295.021985:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202295.022034:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559805440/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202295.022069:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202295.022074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202295.022075:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202295.022081:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1425 00000100:00080000:1.0:1788202295.022087:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202295.022092:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202295.022099:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559805440:192.168.201.156@tcp:250:kworker.0 00000008:00100000:1.0:1788202295.022106:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202295.022124:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559805568/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202295.022139:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202295.022142:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202295.022144:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202295.022148:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1425 00000100:00080000:1.0:1788202295.022159:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202295.022162:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202295.022166:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000800:00000100:0.0:1788202295.022171:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788202295.022177:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559805696/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202295.022223:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202295/real 1788202295] req@ffff93c1c5037800 x1875064559805440/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202295.022239:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559805440/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202295.022250:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202295.022255:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202295.022259:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559805440:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202295.022282:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559805568:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202295.022304:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202295.022313:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202295/real 1788202295] req@ffff93c1c5036680 x1875064559805568/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202295.022332:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559805568/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202295.022341:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202295.022345:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202295.022349:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559805568:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202295.022356:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559805696:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202295.022379:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202295.022389:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202295/real 1788202295] req@ffff93c1c5035c00 x1875064559805696/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202295.022398:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559805696/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202295.022406:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202295.022408:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202295.022411:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559805696:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202295.726002:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202296.749992:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202297.774011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202298.798013:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202299.630049:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202299.630068:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202299.630155:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202299.630173:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202299.630176:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202299.630193:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202299.822012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202300.141976:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202300.141985:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202300.141989:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202300.141998:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1431 00000100:00080000:2.0:1788202300.142008:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202300.142022:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202300.142058:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559805824/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202300.142081:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202300.142084:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202300.142085:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202300.142088:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1431 00000100:00080000:2.0:1788202300.142091:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202300.142094:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202300.142101:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202300.142109:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559805952/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202300.142117:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202300.142120:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202300.142121:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202300.142124:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1431 00000100:00080000:2.0:1788202300.142125:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202300.142128:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202300.142131:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202300.142140:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559806080/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202300.142141:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559805824:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202300.142211:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202300.142254:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202300/real 1788202300] req@ffff93c0e62b8000 x1875064559805824/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202300.142275:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559805824/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202300.142284:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202300.142287:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202300.142291:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559805824:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202300.142296:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559805952:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202300.142311:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202300.142320:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202300/real 1788202300] req@ffff93c0e62ba680 x1875064559805952/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202300.142328:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559805952/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202300.142333:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202300.142336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202300.142338:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559805952:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202300.142358:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559806080:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202300.142370:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202300.142376:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202300/real 1788202300] req@ffff93c0e62b8380 x1875064559806080/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202300.142383:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559806080/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202300.142395:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202300.142397:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202300.142399:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559806080:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788202300.845972:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202301.870013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202302.894070:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202303.918037:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202304.749968:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202304.749987:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202304.750082:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202304.750111:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202304.750116:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202304.750142:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788202304.941988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202305.261954:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202305.261966:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202305.261972:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202305.261982:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1436 00000100:00080000:2.0:1788202305.261994:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202305.262003:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202305.262042:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559806208/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202305.262075:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202305.262079:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202305.262081:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202305.262086:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1436 00000100:00080000:2.0:1788202305.262089:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202305.262094:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202305.262103:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202305.262115:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559806336/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202305.262126:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202305.262129:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202305.262132:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202305.262136:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1436 00000100:00080000:2.0:1788202305.262140:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202305.262146:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202305.262148:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559806208:192.168.201.156@tcp:250:kworker.0 00000008:00100000:2.0:1788202305.262152:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202305.262167:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559806464/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202305.262258:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559806336:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202305.262301:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559806464:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202305.966076:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202306.990031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202308.014023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788202308.334191:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788202308.334253:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788202308.334274:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202308.334326:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202308.334338:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788202308.334443:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202305/real 1788202308] req@ffff93c0e62bb100 x1875064559806464/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202308.334494:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559806464/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202308.334513:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202308.334520:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202308.334526:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559806464:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788202308.334536:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202305/real 1788202308] req@ffff93c0e62bbb80 x1875064559806208/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202308.334551:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559806208/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202308.334581:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202308.334586:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202308.334590:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559806208:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202308.334599:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202305/real 1788202308] req@ffff93c0e62b9c00 x1875064559806336/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202308.334613:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559806336/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202308.334623:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202308.334627:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202308.334631:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559806336:192.168.201.156@tcp:38:kworker.0 00010000:00010000:2.0:1788202309.038028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202309.870012:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202309.870027:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202309.870117:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202309.870149:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202309.870154:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202309.870186:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202310.061966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202310.381971:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202310.381981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202310.381985:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202310.381997:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1441 00000100:00080000:3.0:1788202310.382009:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202310.382015:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202310.382059:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559806592/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202310.382084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202310.382088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202310.382089:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202310.382092:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1441 00000100:00080000:3.0:1788202310.382095:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202310.382098:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202310.382109:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202310.382123:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559806720/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202310.382133:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202310.382138:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202310.382140:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202310.382145:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1441 00000100:00080000:3.0:1788202310.382151:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202310.382156:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202310.382162:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202310.382177:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559806848/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202310.382192:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559806592:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202310.382283:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202310.382331:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202310/real 1788202310] req@ffff93c1c4a4f480 x1875064559806592/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202310.382351:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559806592/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202310.382363:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202310.382368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202310.382373:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559806592:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202310.382381:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559806720:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202310.382407:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202310.382422:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202310/real 1788202310] req@ffff93c1c4a4c700 x1875064559806720/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202310.382434:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559806720/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202310.382443:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202310.382446:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202310.382450:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559806720:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202310.382480:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559806848:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202310.382504:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202310.382517:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202310/real 1788202310] req@ffff93c1c4a4c380 x1875064559806848/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202310.382562:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559806848/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202310.382575:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202310.382579:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202310.382583:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559806848:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788202311.086056:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202312.110001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202313.134089:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202314.157992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202314.990133:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202314.990152:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202314.990264:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202314.990296:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202314.990299:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202314.990323:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788202315.182026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202315.501990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202315.502004:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202315.502009:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202315.502023:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1446 00000100:00080000:3.0:1788202315.502039:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202315.502048:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202315.502098:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559806976/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202315.502131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202315.502135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202315.502138:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202315.502144:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1446 00000100:00080000:3.0:1788202315.502149:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202315.502155:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202315.502168:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202315.502177:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559806976:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202315.502187:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559807104/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202315.502202:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202315.502206:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202315.502208:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202315.502214:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1446 00000100:00080000:3.0:1788202315.502218:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202315.502222:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202315.502228:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202315.502241:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559807232/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202315.502252:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202315.502298:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202315/real 1788202315] req@ffff93c1c4a4ea00 x1875064559806976/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202315.502316:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559806976/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202315.502325:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202315.502329:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202315.502332:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559806976:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202315.502354:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559807104:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202315.502370:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202315.502387:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202315/real 1788202315] req@ffff93c1c4a4c000 x1875064559807104/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202315.502397:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559807104/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202315.502403:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202315.502405:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202315.502408:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559807104:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202315.502412:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559807232:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202315.502431:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202315.502453:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202315/real 1788202315] req@ffff93c1c4a4e680 x1875064559807232/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202315.502463:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559807232/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202315.502480:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202315.502489:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202315.502494:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559807232:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202316.205965:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202317.230041:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202318.253912:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202319.278076:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202320.109976:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202320.109992:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202320.110093:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202320.110121:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202320.110125:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202320.110152:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202320.302005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202320.621964:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202320.621982:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202320.621987:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202320.621998:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1451 00000100:00080000:2.0:1788202320.622009:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202320.622017:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202320.622057:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559807360/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202320.622085:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202320.622089:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202320.622091:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202320.622095:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1451 00000100:00080000:2.0:1788202320.622098:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202320.622103:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202320.622113:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202320.622130:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559807360:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202320.622130:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559807488/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202320.622148:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202320.622153:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202320.622155:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202320.622160:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1451 00000100:00080000:2.0:1788202320.622164:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202320.622168:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202320.622173:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202320.622185:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559807616/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202320.622202:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202320.622244:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202320/real 1788202320] req@ffff93c0e62bad80 x1875064559807360/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202320.622258:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559807360/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202320.622266:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202320.622270:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202320.622274:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559807360:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202320.622296:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559807488:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202320.622312:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202320.622321:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202320/real 1788202320] req@ffff93c0e62b8a80 x1875064559807488/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202320.622329:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559807488/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202320.622339:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202320.622341:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202320.622344:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559807488:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202320.622348:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559807616:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202320.622387:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202320.622396:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202320/real 1788202320] req@ffff93c0e62b9500 x1875064559807616/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202320.622404:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559807616/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202320.622411:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202320.622413:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202320.622415:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559807616:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788202321.326029:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202322.349995:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202323.374026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202324.397978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202325.230019:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202325.230033:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202325.230132:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202325.230160:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202325.230164:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202325.230195:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788202325.422020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202325.741995:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202325.742004:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202325.742008:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202325.742017:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1456 00000100:00080000:2.0:1788202325.742027:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202325.742034:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202325.742072:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559807744/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202325.742106:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202325.742116:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202325.742118:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202325.742123:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1456 00000100:00080000:2.0:1788202325.742127:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202325.742132:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202325.742143:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202325.742154:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559807744:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202325.742161:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559807872/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202325.742174:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202325.742178:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202325.742180:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202325.742185:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1456 00000100:00080000:2.0:1788202325.742189:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202325.742193:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202325.742198:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202325.742212:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559808000/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202325.742229:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202325.742306:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202325/real 1788202325] req@ffff93c0e62b9f80 x1875064559807744/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202325.742326:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559807744/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202325.742337:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202325.742340:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202325.742343:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559807744:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202325.742365:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559807872:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202325.742385:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202325.742395:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202325/real 1788202325] req@ffff93c0e62bb800 x1875064559807872/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202325.742403:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559807872/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202325.742409:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202325.742411:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202325.742413:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559807872:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202325.742417:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559808000:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202325.742432:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202325.742442:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202325/real 1788202325] req@ffff93c0e62b9500 x1875064559808000/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202325.742454:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559808000/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202325.742465:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202325.742468:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202325.742472:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559808000:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202326.446018:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202327.470032:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202328.494035:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202329.518012:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202330.350009:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202330.350023:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202330.350106:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202330.350124:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202330.350127:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202330.350144:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202330.541947:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202330.861905:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202330.861913:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202330.861917:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202330.861925:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1461 00000100:00080000:2.0:1788202330.861934:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202330.861941:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202330.861977:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559808128/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202330.861996:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202330.861999:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202330.862000:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202330.862003:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1461 00000100:00080000:2.0:1788202330.862006:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202330.862008:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202330.862015:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202330.862023:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559808256/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202330.862029:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202330.862032:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202330.862033:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202330.862035:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1461 00000100:00080000:2.0:1788202330.862038:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202330.862040:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202330.862043:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202330.862051:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559808384/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202330.862073:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559808128:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202330.862145:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202330.862209:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202330/real 1788202330] req@ffff93c0e62b8a80 x1875064559808128/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202330.862224:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559808128/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202330.862232:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202330.862235:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202330.862239:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559808128:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202330.862244:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559808256:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202330.862261:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202330.862269:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202330/real 1788202330] req@ffff93c0e62bad80 x1875064559808256/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202330.862277:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559808256/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202330.862282:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202330.862284:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202330.862286:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559808256:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202330.862290:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559808384:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202330.862301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202330.862307:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202330/real 1788202330] req@ffff93c0e62b9c00 x1875064559808384/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202330.862314:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559808384/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202330.862321:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202330.862323:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202330.862324:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559808384:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788202331.566006:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202332.590003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202333.614012:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202334.637983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202335.470020:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202335.470036:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202335.470139:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202335.470161:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202335.470165:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202335.470185:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788202335.661986:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202335.981966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202335.981976:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202335.981979:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202335.981987:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1466 00000100:00080000:1.0:1788202335.981997:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202335.982004:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202335.982041:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559808512/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202335.982063:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202335.982066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202335.982068:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202335.982072:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1466 00000100:00080000:1.0:1788202335.982076:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202335.982080:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202335.982120:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202335.982137:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559808640/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202335.982151:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559808512:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202335.982152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202335.982166:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202335.982169:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202335.982173:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1466 00000100:00080000:1.0:1788202335.982176:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202335.982182:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202335.982189:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202335.982199:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559808768/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202335.982241:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202335.982287:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202335/real 1788202335] req@ffff93c1c5035c00 x1875064559808512/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202335.982302:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559808512/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202335.982313:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202335.982316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202335.982320:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559808512:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202335.982343:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559808640:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202335.982357:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202335.982367:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202335/real 1788202335] req@ffff93c1c5036680 x1875064559808640/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202335.982388:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559808640/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202335.982398:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202335.982402:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202335.982406:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559808640:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202335.982414:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559808768:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202335.982435:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202335.982446:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202335/real 1788202335] req@ffff93c1c5037800 x1875064559808768/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202335.982459:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559808768/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202335.982471:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202335.982475:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202335.982477:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559808768:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202336.685968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202337.709974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202338.733996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202339.758010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202340.590099:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202340.590116:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202340.590219:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202340.590249:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202340.590254:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202340.590284:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202340.782058:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202341.102031:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202341.102045:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202341.102050:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202341.102060:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1471 00000100:00080000:3.0:1788202341.102071:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202341.102079:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202341.102125:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559808896/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202341.102160:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202341.102164:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202341.102166:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202341.102171:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1471 00000100:00080000:3.0:1788202341.102175:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202341.102180:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202341.102190:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202341.102211:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559808896:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202341.102211:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559809024/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202341.102237:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202341.102241:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202341.102243:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202341.102248:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1471 00000100:00080000:3.0:1788202341.102253:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202341.102258:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202341.102264:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000800:00000100:0.0:1788202341.102280:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202341.102282:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559809152/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202341.102318:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202341/real 1788202341] req@ffff93c1c4a4fb80 x1875064559808896/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202341.102338:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559808896/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202341.102346:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202341.102350:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202341.102353:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559808896:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202341.102372:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559809024:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202341.102388:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202341.102397:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202341/real 1788202341] req@ffff93c1c4a4dc00 x1875064559809024/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202341.102405:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559809024/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202341.102410:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202341.102412:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202341.102414:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559809024:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202341.102418:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559809152:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202341.102428:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202341.102434:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202341/real 1788202341] req@ffff93c1c4a4ed80 x1875064559809152/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202341.102441:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559809152/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202341.102452:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202341.102454:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202341.102456:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559809152:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788202341.806035:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202342.830003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788202343.854064:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202344.877981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202345.710078:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202345.710095:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202345.710210:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202345.710233:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202345.710236:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202345.710254:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202345.902098:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202346.222001:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202346.222011:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202346.222016:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202346.222027:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1477 00000100:00080000:2.0:1788202346.222038:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202346.222045:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202346.222089:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559809280/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202346.222116:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202346.222119:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202346.222121:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202346.222125:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1477 00000100:00080000:2.0:1788202346.222127:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202346.222132:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202346.222146:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202346.222162:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559809408/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202346.222167:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559809280:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202346.222185:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202346.222189:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202346.222190:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202346.222196:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1477 00000100:00080000:2.0:1788202346.222199:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202346.222202:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202346.222207:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202346.222218:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559809536/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202346.222242:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202346.222284:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202346/real 1788202346] req@ffff93c0e62bbb80 x1875064559809280/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202346.222299:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559809280/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202346.222310:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202346.222320:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202346.222325:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559809280:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202346.222355:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559809408:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202346.222377:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202346.222387:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202346/real 1788202346] req@ffff93c0e62bb100 x1875064559809408/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202346.222402:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559809408/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202346.222409:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202346.222411:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202346.222414:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559809408:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202346.222418:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559809536:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202346.222432:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202346.222439:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202346/real 1788202346] req@ffff93c0e62b8380 x1875064559809536/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202346.222447:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559809536/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202346.222455:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202346.222457:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202346.222459:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559809536:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202346.925968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202347.950028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202348.974016:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202349.998060:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202350.829988:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202350.830003:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202350.830103:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202350.830124:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202350.830127:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202350.830156:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202351.021917:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202351.341985:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202351.341995:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202351.341999:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202351.342007:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1482 00000100:00080000:0.0:1788202351.342018:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202351.342024:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202351.342064:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559809664/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202351.342096:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202351.342101:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202351.342103:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202351.342107:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1482 00000100:00080000:0.0:1788202351.342112:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202351.342117:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202351.342128:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202351.342145:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559809792/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788202351.342146:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559809664:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202351.342166:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202351.342171:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202351.342173:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202351.342178:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1482 00000100:00080000:0.0:1788202351.342182:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202351.342187:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202351.342192:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202351.342205:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559809920/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202351.342226:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202351.342270:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202351/real 1788202351] req@ffff93c0e9ce4a80 x1875064559809664/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202351.342291:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559809664/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202351.342302:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202351.342306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202351.342311:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559809664:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202351.342336:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559809792:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202351.342360:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202351.342374:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202351/real 1788202351] req@ffff93c0e9ce5f80 x1875064559809792/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202351.342387:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559809792/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202351.342396:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202351.342399:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202351.342402:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559809792:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202351.342409:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559809920:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202351.342429:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202351.342440:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202351/real 1788202351] req@ffff93c0e9ce7100 x1875064559809920/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202351.342451:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559809920/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202351.342462:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202351.342466:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202351.342469:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559809920:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202352.046030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202353.069996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202354.094072:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202355.117933:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202355.949985:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202355.950000:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202355.950093:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202355.950113:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202355.950116:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202355.950134:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202356.142028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202356.461969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202356.461978:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202356.461982:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202356.461990:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1487 00000100:00080000:3.0:1788202356.461999:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202356.462006:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202356.462041:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f100 x1875064559810048/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202356.462066:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202356.462069:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202356.462070:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202356.462073:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1487 00000100:00080000:3.0:1788202356.462075:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202356.462078:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202356.462086:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202356.462095:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4df80 x1875064559810176/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202356.462102:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202356.462105:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202356.462106:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202356.462109:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1487 00000100:00080000:3.0:1788202356.462111:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202356.462113:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202356.462119:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559810048:192.168.201.156@tcp:250:kworker.0 00000008:00100000:3.0:1788202356.462120:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202356.462132:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559810304/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202356.462186:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202356.462220:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202356/real 1788202356] req@ffff93c1c4a4f100 x1875064559810048/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202356.462239:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f100 x1875064559810048/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202356.462247:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202356.462249:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202356.462252:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559810048:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202356.462258:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559810176:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202356.462273:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202356.462281:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202356/real 1788202356] req@ffff93c1c4a4df80 x1875064559810176/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202356.462289:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4df80 x1875064559810176/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202356.462294:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202356.462297:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202356.462299:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559810176:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202356.462318:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559810304:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202356.462335:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202356.462347:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202356/real 1788202356] req@ffff93c1c4a4ed80 x1875064559810304/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202356.462360:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559810304/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202356.462377:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202356.462381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202356.462384:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559810304:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202357.165943:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202358.190034:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202359.214032:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202360.238018:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202361.069992:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202361.070009:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202361.070118:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202361.070141:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202361.070144:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202361.070161:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202361.261996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202361.581972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202361.581985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202361.581990:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202361.582001:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1492 00000100:00080000:1.0:1788202361.582011:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202361.582017:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202361.582052:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559810432/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202361.582074:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202361.582077:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202361.582078:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202361.582082:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1492 00000100:00080000:1.0:1788202361.582084:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202361.582086:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202361.582097:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202361.582105:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559810560/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202361.582113:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202361.582115:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202361.582116:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202361.582120:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1492 00000100:00080000:1.0:1788202361.582123:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788202361.582125:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559810432:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202361.582126:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202361.582132:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202361.582139:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559810688/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202361.582199:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202361.582236:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202361/real 1788202361] req@ffff93c1c5037100 x1875064559810432/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202361.582251:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559810432/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202361.582259:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202361.582262:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202361.582264:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559810432:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202361.582285:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559810560:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202361.582301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202361.582312:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202361/real 1788202361] req@ffff93c1c5034e00 x1875064559810560/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202361.582325:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559810560/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202361.582334:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202361.582337:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202361.582341:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559810560:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202361.582349:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559810688:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202361.582370:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202361.582383:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202361/real 1788202361] req@ffff93c1c5034a80 x1875064559810688/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202361.582396:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559810688/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202361.582407:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202361.582410:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202361.582414:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559810688:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202362.286055:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202363.310018:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202364.334052:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202365.357982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202366.190023:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202366.190040:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202366.190162:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202366.190187:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202366.190190:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202366.190209:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202366.382073:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202366.701969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202366.701982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202366.701988:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202366.702001:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1497 00000100:00080000:1.0:1788202366.702016:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202366.702026:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202366.702091:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559810816/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202366.702135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202366.702139:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202366.702142:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202366.702147:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1497 00000100:00080000:1.0:1788202366.702151:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202366.702156:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202366.702168:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202366.702185:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559810944/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788202366.702192:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559810816:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202366.702205:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202366.702210:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202366.702213:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202366.702219:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1497 00000100:00080000:1.0:1788202366.702223:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202366.702228:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202366.702235:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202366.702249:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559811072/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202366.702265:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202366.702320:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202366/real 1788202366] req@ffff93c1c5034a80 x1875064559810816/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202366.702336:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559810816/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202366.702346:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202366.702351:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202366.702356:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559810816:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202366.702386:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559810944:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202366.702411:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202366.702422:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202366/real 1788202366] req@ffff93c1c5034e00 x1875064559810944/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202366.702436:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559810944/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202366.702447:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202366.702451:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202366.702455:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559810944:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202366.702462:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559811072:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202366.702484:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202366.702493:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202366/real 1788202366] req@ffff93c1c5037100 x1875064559811072/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202366.702502:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559811072/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202366.702510:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202366.702513:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202366.702515:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559811072:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202367.405980:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202368.429997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202369.454009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202370.477985:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202371.310023:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202371.310051:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202371.310163:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202371.310184:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202371.310187:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202371.310205:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202371.502017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202371.821937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202371.821946:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202371.821950:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202371.821961:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1502 00000100:00080000:2.0:1788202371.821972:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202371.821980:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202371.822021:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559811200/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202371.822048:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202371.822051:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202371.822053:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202371.822056:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1502 00000100:00080000:2.0:1788202371.822059:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202371.822061:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202371.822071:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202371.822082:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559811328/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202371.822091:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202371.822095:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788202371.822096:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559811200:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202371.822096:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202371.822103:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1502 00000100:00080000:2.0:1788202371.822108:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202371.822111:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202371.822114:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202371.822121:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559811456/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202371.822227:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559811328:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202371.822274:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559811456:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202372.525989:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202373.550036:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202374.573978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788202374.894282:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788202374.894350:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788202374.894372:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788202374.894433:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788202374.894446:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788202374.894668:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202371/real 1788202374] req@ffff93c0e62b8000 x1875064559811328/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202374.894708:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559811328/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202374.894723:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202374.894729:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202374.894734:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559811328:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202374.894744:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202371/real 1788202374] req@ffff93c0e62b8700 x1875064559811456/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202374.894759:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559811456/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202374.894771:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202374.894775:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202374.894779:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559811456:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788202374.894787:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202371/real 1788202374] req@ffff93c0e62ba680 x1875064559811200/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202374.894801:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559811200/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202374.894810:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202374.894814:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202374.894818:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559811200:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788202375.598001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202376.429996:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202376.430013:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202376.430120:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202376.430149:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202376.430154:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202376.430188:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202376.622039:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202376.941968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202376.941982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202376.941987:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202376.941999:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1507 00000100:00080000:2.0:1788202376.942010:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202376.942018:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202376.942064:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559811584/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202376.942108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202376.942113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202376.942115:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202376.942120:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1507 00000100:00080000:2.0:1788202376.942124:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202376.942128:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202376.942138:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202376.942151:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559811712/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202376.942165:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202376.942168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202376.942170:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202376.942173:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1507 00000100:00080000:2.0:1788202376.942176:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202376.942179:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202376.942182:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202376.942196:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559811840/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202376.942213:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559811584:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202376.942320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202376.942367:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202376/real 1788202376] req@ffff93c0e62baa00 x1875064559811584/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202376.942403:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559811584/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202376.942416:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202376.942420:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202376.942425:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559811584:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202376.942434:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559811712:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202376.942460:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202376.942491:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202376/real 1788202376] req@ffff93c0e62bb480 x1875064559811712/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202376.942504:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559811712/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202376.942513:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202376.942517:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202376.942520:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559811712:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202376.942552:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559811840:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202376.942577:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202376.942590:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202376/real 1788202376] req@ffff93c0e62ba680 x1875064559811840/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202376.942610:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559811840/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202376.942621:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202376.942625:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202376.942628:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559811840:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202377.645978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202378.670033:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202379.693983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202380.718062:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202381.550018:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202381.550036:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202381.550148:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202381.550167:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202381.550170:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202381.550187:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202381.742021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202382.061988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202382.061999:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202382.062003:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202382.062015:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1512 00000100:00080000:2.0:1788202382.062028:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202382.062035:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202382.062086:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559811968/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202382.062126:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202382.062129:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202382.062131:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202382.062135:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1512 00000100:00080000:2.0:1788202382.062139:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202382.062142:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202382.062158:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202382.062159:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559811968:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202382.062172:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559812096/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202382.062182:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202382.062185:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202382.062186:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202382.062189:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1512 00000100:00080000:2.0:1788202382.062192:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202382.062195:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202382.062199:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202382.062207:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559812224/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202382.062234:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202382.062276:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202382/real 1788202382] req@ffff93c0e62b8700 x1875064559811968/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202382.062311:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559811968/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202382.062324:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202382.062328:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202382.062332:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559811968:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202382.062357:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559812096:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202382.062377:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202382.062387:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202382/real 1788202382] req@ffff93c0e62b8000 x1875064559812096/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202382.062401:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559812096/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202382.062407:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202382.062409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202382.062412:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559812096:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202382.062416:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559812224:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202382.062428:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202382.062434:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202382/real 1788202382] req@ffff93c0e62b8380 x1875064559812224/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202382.062442:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559812224/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202382.062449:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202382.062451:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202382.062453:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559812224:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202382.766038:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202383.790018:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202384.814031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202385.837997:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202386.670027:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202386.670043:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202386.670142:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202386.670167:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202386.670170:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202386.670190:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202386.861996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202387.181901:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202387.181910:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202387.181914:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202387.181922:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1518 00000100:00080000:2.0:1788202387.181932:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202387.181939:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202387.181973:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559812352/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202387.182003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202387.182006:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202387.182007:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202387.182010:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1518 00000100:00080000:2.0:1788202387.182013:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202387.182015:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202387.182023:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202387.182032:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559812480/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202387.182039:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202387.182042:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202387.182044:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202387.182047:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1518 00000100:00080000:2.0:1788202387.182049:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202387.182054:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202387.182057:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559812352:192.168.201.156@tcp:250:kworker.0 00000008:00100000:2.0:1788202387.182060:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202387.182071:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559812608/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202387.182122:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202387.182174:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202387/real 1788202387] req@ffff93c0e62bb100 x1875064559812352/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202387.182191:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559812352/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202387.182214:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202387.182217:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202387.182221:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559812352:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202387.182236:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559812480:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202387.182257:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202387.182267:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202387/real 1788202387] req@ffff93c0e62bbb80 x1875064559812480/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202387.182275:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559812480/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202387.182281:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202387.182283:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202387.182286:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559812480:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202387.182305:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559812608:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202387.182316:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202387.182322:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202387/real 1788202387] req@ffff93c0e62b9c00 x1875064559812608/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202387.182330:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559812608/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202387.182341:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202387.182343:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202387.182345:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559812608:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202387.885972:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202388.910048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202389.933973:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202390.958011:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202391.790020:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202391.790034:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202391.790133:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202391.790163:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202391.790167:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202391.790218:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202391.981966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202392.302000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202392.302008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202392.302012:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202392.302020:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1523 00000100:00080000:1.0:1788202392.302033:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202392.302040:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202392.302075:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559812736/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202392.302100:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202392.302102:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202392.302104:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202392.302132:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1523 00000100:00080000:1.0:1788202392.302136:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202392.302140:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202392.302149:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202392.302159:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559812864/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202392.302167:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202392.302170:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202392.302171:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202392.302174:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1523 00000100:00080000:1.0:1788202392.302177:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202392.302180:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202392.302184:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202392.302192:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559812992/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202392.302202:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559812736:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202392.302306:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202392.302379:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202392/real 1788202392] req@ffff93c1c5037800 x1875064559812736/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202392.302405:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559812736/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202392.302420:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202392.302426:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202392.302431:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559812736:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202392.302441:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559812864:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202392.302469:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202392.302484:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202392/real 1788202392] req@ffff93c1c5036680 x1875064559812864/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202392.302499:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559812864/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202392.302509:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202392.302513:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202392.302517:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559812864:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202392.302550:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559812992:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202392.302574:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202392.302587:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202392/real 1788202392] req@ffff93c1c5035c00 x1875064559812992/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202392.302600:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559812992/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202392.302612:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202392.302616:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202392.302620:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559812992:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202393.005983:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202394.030131:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202395.053993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202396.078005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202396.910014:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202396.910041:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202396.910143:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202396.910165:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202396.910169:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202396.910189:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202397.101988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202397.421958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202397.421966:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202397.421970:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202397.421979:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1528 00000100:00080000:1.0:1788202397.421989:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202397.421995:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202397.422056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559813120/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202397.422079:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202397.422082:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202397.422083:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202397.422086:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1528 00000100:00080000:1.0:1788202397.422089:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202397.422091:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202397.422101:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202397.422110:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559813248/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202397.422118:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202397.422121:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202397.422122:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202397.422126:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1528 00000100:00100000:0.0:1788202397.422128:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559813120:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202397.422128:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202397.422134:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202397.422140:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202397.422148:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559813376/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202397.422194:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202397.422228:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202397/real 1788202397] req@ffff93c1c5035c00 x1875064559813120/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202397.422250:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559813120/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202397.422258:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202397.422261:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202397.422264:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559813120:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202397.422283:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559813248:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202397.422299:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202397.422314:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202397/real 1788202397] req@ffff93c1c5036680 x1875064559813248/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202397.422323:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559813248/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202397.422328:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202397.422330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202397.422333:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559813248:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202397.422337:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559813376:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202397.422348:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202397.422354:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202397/real 1788202397] req@ffff93c1c5037800 x1875064559813376/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202397.422361:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559813376/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202397.422368:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202397.422370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202397.422372:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559813376:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202398.126002:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202399.149976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202400.174016:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202401.198036:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202402.030954:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202402.030968:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202402.031077:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202402.031107:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202402.031112:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202402.031153:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202402.222027:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202402.541975:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202402.541984:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202402.541988:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202402.541997:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1533 00000100:00080000:2.0:1788202402.542007:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202402.542013:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202402.542047:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559813504/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202402.542073:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202402.542075:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202402.542083:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202402.542087:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1533 00000100:00080000:2.0:1788202402.542089:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202402.542092:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202402.542100:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202402.542109:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559813632/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202402.542116:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202402.542119:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202402.542120:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202402.542123:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1533 00000100:00080000:2.0:1788202402.542125:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202402.542131:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202402.542134:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559813504:192.168.201.156@tcp:250:kworker.0 00000008:00100000:2.0:1788202402.542136:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202402.542145:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559813760/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202402.542201:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202402.542244:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202402/real 1788202402] req@ffff93c0e62bad80 x1875064559813504/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202402.542264:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559813504/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202402.542288:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202402.542293:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202402.542297:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559813504:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202402.542304:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559813632:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202402.542323:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202402.542333:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202402/real 1788202402] req@ffff93c0e62b8a80 x1875064559813632/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202402.542347:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559813632/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202402.542353:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202402.542355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202402.542357:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559813632:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202402.542375:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559813760:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202402.542386:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202402.542393:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202402/real 1788202402] req@ffff93c0e62b9500 x1875064559813760/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202402.542400:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559813760/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202402.542407:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202402.542409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202402.542411:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559813760:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202403.245959:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202404.270006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202405.294037:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202406.318006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202407.150045:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202407.150059:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202407.150149:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202407.150168:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202407.150171:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202407.150195:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202407.341961:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202407.661967:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202407.661977:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202407.661981:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202407.661989:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1538 00000100:00080000:2.0:1788202407.661999:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202407.662005:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202407.662041:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559813888/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202407.662065:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202407.662069:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202407.662071:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202407.662075:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1538 00000100:00080000:2.0:1788202407.662079:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202407.662083:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202407.662094:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202407.662120:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559814016/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202407.662136:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202407.662139:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202407.662141:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202407.662144:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1538 00000100:00080000:2.0:1788202407.662147:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202407.662151:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202407.662157:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202407.662162:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559813888:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202407.662179:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559814144/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202407.662249:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202407.662305:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202407/real 1788202407] req@ffff93c0e62bb800 x1875064559813888/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202407.662321:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559813888/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202407.662331:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202407.662335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202407.662339:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559813888:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202407.662344:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559814016:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202407.662360:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202407.662368:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202407/real 1788202407] req@ffff93c0e62b9f80 x1875064559814016/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202407.662377:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559814016/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202407.662382:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202407.662385:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202407.662387:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559814016:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202407.662412:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559814144:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202407.662424:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202407.662430:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202407/real 1788202407] req@ffff93c0e62b9500 x1875064559814144/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202407.662438:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559814144/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202407.662451:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202407.662453:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202407.662455:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559814144:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202408.366005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202409.389978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202410.413982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202411.437995:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202412.270054:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202412.270071:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202412.270165:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202412.270192:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202412.270196:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202412.270222:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202412.461998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202412.782015:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202412.782024:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202412.782028:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202412.782036:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1543 00000100:00080000:1.0:1788202412.782047:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202412.782054:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202412.782100:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559814272/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202412.782134:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202412.782137:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202412.782139:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202412.782142:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1543 00000100:00080000:1.0:1788202412.782145:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202412.782148:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202412.782157:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202412.782169:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559814400/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202412.782178:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202412.782181:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202412.782182:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202412.782191:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1543 00000100:00080000:1.0:1788202412.782193:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202412.782196:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202412.782201:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202412.782210:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559814528/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202412.782214:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559814272:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202412.782317:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202412.782379:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202412/real 1788202412] req@ffff93c1c5037100 x1875064559814272/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202412.782403:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559814272/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202412.782417:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202412.782423:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202412.782427:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559814272:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202412.782436:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559814400:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202412.782460:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202412.782473:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202412/real 1788202412] req@ffff93c1c5034e00 x1875064559814400/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202412.782486:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559814400/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202412.782496:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202412.782499:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202412.782503:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559814400:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202412.782533:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559814528:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202412.782553:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202412.782564:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202412/real 1788202412] req@ffff93c1c5034a80 x1875064559814528/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202412.782576:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559814528/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202412.782587:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202412.782590:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202412.782594:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559814528:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202413.485997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202414.509986:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202415.533992:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202416.557993:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202417.390006:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202417.390025:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202417.390152:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202417.390179:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202417.390184:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202417.390212:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202417.582005:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202417.901990:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202417.902005:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202417.902010:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202417.902024:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1548 00000100:00080000:2.0:1788202417.902039:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202417.902047:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202417.902102:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559814656/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202417.902137:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202417.902142:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202417.902144:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202417.902150:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1548 00000100:00080000:2.0:1788202417.902154:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202417.902158:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202417.902171:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202417.902187:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559814784/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202417.902193:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559814656:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202417.902208:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202417.902213:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202417.902215:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202417.902222:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1548 00000100:00080000:2.0:1788202417.902227:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202417.902231:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202417.902238:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202417.902251:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559814912/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202417.902276:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202417.902318:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202417/real 1788202417] req@ffff93c0e62b8a80 x1875064559814656/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202417.902333:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559814656/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202417.902343:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202417.902346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202417.902350:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559814656:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202417.902370:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559814784:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202417.902386:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202417.902401:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202417/real 1788202417] req@ffff93c0e62bad80 x1875064559814784/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202417.902410:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559814784/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202417.902416:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202417.902419:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202417.902421:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559814784:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202417.902425:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559814912:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202417.902439:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202417.902445:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202417/real 1788202417] req@ffff93c0e62b9c00 x1875064559814912/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202417.902453:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559814912/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202417.902462:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202417.902465:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202417.902469:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559814912:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202418.605995:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202419.629997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202420.653982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202421.677942:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202422.510099:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202422.510120:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202422.510259:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202422.510287:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202422.510291:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202422.510309:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202422.701973:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202423.021934:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202423.021944:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202423.021948:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202423.021958:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1553 00000100:00080000:1.0:1788202423.021970:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202423.021977:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202423.022018:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559815040/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202423.022052:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202423.022057:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202423.022064:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202423.022069:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1553 00000100:00080000:1.0:1788202423.022073:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202423.022077:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202423.022088:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202423.022102:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559815168/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202423.022111:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559815040:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202423.022113:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202423.022118:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202423.022121:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202423.022126:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1553 00000100:00080000:1.0:1788202423.022129:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202423.022132:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202423.022135:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202423.022144:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559815296/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202423.022195:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202423.022253:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202423/real 1788202423] req@ffff93c1c5034a80 x1875064559815040/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202423.022275:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559815040/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202423.022288:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202423.022294:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202423.022299:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559815040:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202423.022328:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559815168:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202423.022349:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202423.022359:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202423/real 1788202423] req@ffff93c1c5034e00 x1875064559815168/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202423.022368:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559815168/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202423.022374:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202423.022376:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202423.022379:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559815168:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202423.022384:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559815296:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202423.022396:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202423.022402:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202423/real 1788202423] req@ffff93c1c5037100 x1875064559815296/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202423.022410:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559815296/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202423.022418:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202423.022420:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202423.022422:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559815296:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202423.726017:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202424.750009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202425.774022:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202426.797964:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202427.630001:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202427.630018:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202427.630126:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202427.630162:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202427.630167:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202427.630194:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202427.822021:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202428.141900:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202428.141914:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202428.141919:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202428.141932:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1559 00000100:00080000:1.0:1788202428.141944:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202428.141953:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202428.141998:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559815424/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202428.142032:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202428.142036:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202428.142038:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202428.142042:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1559 00000100:00080000:1.0:1788202428.142045:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202428.142050:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202428.142062:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202428.142076:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559815424:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788202428.142078:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559815552/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202428.142092:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202428.142096:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202428.142098:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202428.142103:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1559 00000100:00080000:1.0:1788202428.142106:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202428.142111:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202428.142116:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202428.142128:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559815680/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202428.142141:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202428.142187:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202428/real 1788202428] req@ffff93c1c5037800 x1875064559815424/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202428.142208:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559815424/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202428.142217:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202428.142221:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202428.142225:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559815424:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202428.142246:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559815552:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202428.142261:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202428.142273:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202428/real 1788202428] req@ffff93c1c5036680 x1875064559815552/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202428.142286:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559815552/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202428.142291:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202428.142294:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202428.142296:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559815552:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202428.142299:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559815680:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202428.142311:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202428.142316:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202428/real 1788202428] req@ffff93c1c5035c00 x1875064559815680/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202428.142323:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559815680/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202428.142330:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202428.142332:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202428.142334:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559815680:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202428.845935:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202429.869998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202430.893959:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202431.917971:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202432.749999:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202432.750014:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202432.750117:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202432.750145:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202432.750150:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202432.750180:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202432.941978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202433.261983:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202433.261994:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202433.261998:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202433.262009:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1564 00000100:00080000:2.0:1788202433.262027:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202433.262034:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202433.262086:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559815808/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202433.262114:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202433.262117:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202433.262118:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202433.262122:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1564 00000100:00080000:2.0:1788202433.262125:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202433.262134:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202433.262144:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202433.262156:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559815936/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202433.262165:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202433.262169:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788202433.262171:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559815808:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202433.262172:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202433.262184:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1564 00000100:00080000:2.0:1788202433.262189:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202433.262193:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202433.262200:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202433.262214:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559816064/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202433.262249:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202433.262293:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202433/real 1788202433] req@ffff93c0e62bbb80 x1875064559815808/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202433.262315:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559815808/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202433.262328:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202433.262333:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202433.262337:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559815808:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202433.262366:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559815936:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202433.262390:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202433.262425:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202433/real 1788202433] req@ffff93c0e62bb100 x1875064559815936/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202433.262438:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559815936/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202433.262447:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202433.262451:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202433.262455:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559815936:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202433.262461:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559816064:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202433.262481:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202433.262492:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202433/real 1788202433] req@ffff93c0e62b8380 x1875064559816064/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202433.262505:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559816064/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202433.262516:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202433.262520:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202433.262523:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559816064:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202433.965986:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202434.989942:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202436.013975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202437.038005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202437.870078:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202437.870098:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202437.870244:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202437.870275:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202437.870279:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202437.870303:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202438.061978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202438.381984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202438.381994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202438.381998:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202438.382014:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1569 00000100:00080000:0.0:1788202438.382026:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202438.382033:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202438.382080:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559816192/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202438.382111:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202438.382113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202438.382115:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202438.382118:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1569 00000100:00080000:0.0:1788202438.382120:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202438.382123:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202438.382134:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202438.382145:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559816320/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788202438.382150:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559816192:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202438.382154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202438.382158:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202438.382160:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202438.382163:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1569 00000100:00080000:0.0:1788202438.382165:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202438.382167:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202438.382171:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202438.382180:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559816448/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788202438.382273:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559816320:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202438.382295:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559816448:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202439.085948:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202440.110021:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202441.134023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788202441.454102:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788202441.454148:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788202441.454167:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202441.454207:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202441.454213:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:2.0:1788202441.454265:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202438/real 1788202441] req@ffff93c0e9ce6d80 x1875064559816320/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202441.454295:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559816320/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202441.454305:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202441.454311:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202441.454316:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559816320:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202441.454325:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202438/real 1788202441] req@ffff93c0e9ce5c00 x1875064559816448/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202441.454339:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559816448/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202441.454350:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202441.454354:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202441.454358:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559816448:192.168.201.156@tcp:8:kworker.0 00000100:00100000:2.0:1788202441.454365:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202438/real 1788202441] req@ffff93c0e9ce7b80 x1875064559816192/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202441.454378:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559816192/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202441.454386:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202441.454390:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202441.454393:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559816192:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788202442.157963:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202442.990004:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202442.990020:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202442.990127:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202442.990158:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202442.990163:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202442.990196:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202443.182021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202443.501952:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202443.501962:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202443.501966:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202443.501974:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1574 00000100:00080000:1.0:1788202443.501984:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202443.501995:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202443.502027:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559816576/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202443.502050:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202443.502053:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202443.502055:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202443.502059:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1574 00000100:00080000:1.0:1788202443.502063:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202443.502067:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202443.502077:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202443.502093:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559816704/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202443.502107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788202443.502109:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559816576:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202443.502112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202443.502116:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202443.502123:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1574 00000100:00080000:1.0:1788202443.502136:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202443.502141:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202443.502147:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202443.502162:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559816832/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202443.502191:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202443.502262:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202443/real 1788202443] req@ffff93c1c5035c00 x1875064559816576/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202443.502284:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559816576/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202443.502294:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202443.502298:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202443.502302:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559816576:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202443.502328:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559816704:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202443.502354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202443.502368:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202443/real 1788202443] req@ffff93c1c5036680 x1875064559816704/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202443.502387:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559816704/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202443.502396:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202443.502400:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202443.502403:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559816704:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202443.502408:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559816832:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202443.502429:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202443.502452:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202443/real 1788202443] req@ffff93c1c5037800 x1875064559816832/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202443.502465:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559816832/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202443.502474:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202443.502478:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202443.502481:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559816832:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202444.205975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202445.229989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202446.253961:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202447.278020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202448.109967:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202448.109982:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202448.110068:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202448.110088:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202448.110090:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202448.110107:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202448.301958:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202448.621955:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202448.621964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202448.621968:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202448.621977:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1579 00000100:00080000:1.0:1788202448.621987:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202448.621994:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202448.622029:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559816960/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202448.622057:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202448.622060:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202448.622062:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202448.622065:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1579 00000100:00080000:1.0:1788202448.622067:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202448.622070:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202448.622077:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202448.622087:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559817088/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788202448.622096:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559816960:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202448.622108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202448.622111:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202448.622113:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202448.622117:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1579 00000100:00080000:1.0:1788202448.622119:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202448.622122:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202448.622125:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202448.622148:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559817216/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202448.622183:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202448.622223:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202448/real 1788202448] req@ffff93c1c5037100 x1875064559816960/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202448.622238:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559816960/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202448.622251:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202448.622253:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202448.622257:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559816960:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202448.622287:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559817088:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202448.622305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202448.622314:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202448/real 1788202448] req@ffff93c1c5034e00 x1875064559817088/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202448.622322:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559817088/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202448.622328:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202448.622330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202448.622332:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559817088:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202448.622336:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559817216:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202448.622346:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202448.622352:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202448/real 1788202448] req@ffff93c1c5034a80 x1875064559817216/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202448.622359:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559817216/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202448.622381:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202448.622384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202448.622388:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559817216:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202449.325991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202450.350015:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202451.373985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202452.398028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202453.230009:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202453.230023:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202453.230138:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202453.230161:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202453.230164:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202453.230183:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202453.421981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202453.741991:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202453.742002:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202453.742006:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202453.742014:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1584 00000100:00080000:2.0:1788202453.742024:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202453.742030:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202453.742084:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559817344/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202453.742112:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202453.742114:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202453.742116:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202453.742118:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1584 00000100:00080000:2.0:1788202453.742121:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202453.742124:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202453.742131:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202453.742139:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559817472/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202453.742146:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202453.742149:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202453.742150:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202453.742153:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1584 00000100:00080000:2.0:1788202453.742155:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202453.742158:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202453.742162:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202453.742169:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559817600/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202453.742178:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559817344:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202453.742253:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202453.742312:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202453/real 1788202453] req@ffff93c0e62b8000 x1875064559817344/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202453.742333:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559817344/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202453.742344:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202453.742348:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202453.742352:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559817344:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202453.742360:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559817472:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202453.742386:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202453.742400:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202453/real 1788202453] req@ffff93c0e62b8700 x1875064559817472/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202453.742413:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559817472/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202453.742428:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202453.742432:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202453.742436:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559817472:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202453.742462:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559817600:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202453.742484:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202453.742494:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202453/real 1788202453] req@ffff93c0e62ba680 x1875064559817600/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202453.742502:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559817600/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202453.742509:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202453.742511:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202453.742513:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559817600:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202454.445972:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202455.470003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202456.494017:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202457.517994:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202458.350037:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202458.350053:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202458.350137:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202458.350156:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202458.350159:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202458.350179:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202458.542008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202458.861947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202458.861977:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202458.861982:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202458.861993:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1589 00000100:00080000:0.0:1788202458.862005:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202458.862014:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202458.862055:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559817728/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202458.862081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202458.862083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202458.862085:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202458.862088:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1589 00000100:00080000:0.0:1788202458.862092:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202458.862096:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202458.862105:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202458.862112:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559817728:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202458.862118:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559817856/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202458.862127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202458.862129:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202458.862131:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202458.862134:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1589 00000100:00080000:0.0:1788202458.862136:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202458.862138:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202458.862142:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202458.862151:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559817984/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202458.862184:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202458.862218:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202458/real 1788202458] req@ffff93c0e9ce6680 x1875064559817728/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202458.862233:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559817728/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202458.862241:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202458.862244:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202458.862248:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559817728:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202458.862267:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559817856:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202458.862282:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202458.862290:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202458/real 1788202458] req@ffff93c0e9ce4000 x1875064559817856/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202458.862298:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559817856/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202458.862304:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202458.862306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202458.862308:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559817856:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202458.862312:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559817984:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202458.862323:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202458.862329:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202458/real 1788202458] req@ffff93c0e9ce6a00 x1875064559817984/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202458.862336:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559817984/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202458.862343:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202458.862345:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202458.862346:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559817984:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202459.566017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202460.589998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202461.614011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202462.637995:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202463.470001:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202463.470015:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202463.470113:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202463.470143:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202463.470147:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202463.470176:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202463.661947:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202463.981950:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202463.981959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202463.981963:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202463.981971:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1594 00000100:00080000:0.0:1788202463.981981:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202463.981987:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202463.982021:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559818112/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202463.982044:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202463.982046:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202463.982047:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202463.982050:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1594 00000100:00080000:0.0:1788202463.982052:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202463.982055:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202463.982063:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202463.982072:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559818240/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202463.982079:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202463.982081:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202463.982082:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202463.982085:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1594 00000100:00080000:0.0:1788202463.982087:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202463.982089:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202463.982093:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202463.982099:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559818112:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202463.982103:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559818368/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202463.982163:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202463.982207:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202463/real 1788202463] req@ffff93c0e9ce4380 x1875064559818112/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202463.982227:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559818112/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202463.982238:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202463.982242:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202463.982247:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559818112:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202463.982256:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559818240:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202463.982281:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202463.982295:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202463/real 1788202463] req@ffff93c0e9ce4700 x1875064559818240/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202463.982316:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559818240/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202463.982326:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202463.982328:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202463.982330:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559818240:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202463.982351:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559818368:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202463.982374:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202463.982383:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202463/real 1788202463] req@ffff93c0e9ce7480 x1875064559818368/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202463.982391:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559818368/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202463.982398:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202463.982400:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202463.982404:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559818368:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202464.686012:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202465.709958:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202466.733969:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202467.757958:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202468.590003:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202468.590019:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202468.590122:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202468.590150:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202468.590155:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202468.590183:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202468.781982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202469.101967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202469.101977:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202469.101981:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202469.101990:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1599 00000100:00080000:1.0:1788202469.102000:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202469.102011:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202469.102047:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559818496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202469.102070:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202469.102073:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202469.102074:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202469.102077:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1599 00000100:00080000:1.0:1788202469.102079:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202469.102082:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202469.102089:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202469.102097:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559818624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202469.102105:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202469.102107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202469.102109:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202469.102112:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1599 00000100:00080000:1.0:1788202469.102114:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202469.102117:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202469.102122:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202469.102123:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559818496:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788202469.102132:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559818752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202469.102185:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202469.102225:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202469/real 1788202469] req@ffff93c1c5034a80 x1875064559818496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202469.102245:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559818496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202469.102259:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202469.102263:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202469.102268:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559818496:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202469.102276:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559818624:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202469.102302:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202469.102315:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202469/real 1788202469] req@ffff93c1c5034e00 x1875064559818624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202469.102323:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559818624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202469.102329:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202469.102331:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202469.102333:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559818624:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202469.102353:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559818752:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202469.102367:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202469.102373:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202469/real 1788202469] req@ffff93c1c5037100 x1875064559818752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202469.102380:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559818752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202469.102387:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202469.102389:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202469.102391:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559818752:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202469.806030:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202470.830030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202471.854003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202472.878030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202473.710012:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202473.710034:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202473.710167:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202473.710193:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202473.710196:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202473.710221:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202473.902071:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202474.222011:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202474.222025:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202474.222031:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202474.222044:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1605 00000100:00080000:1.0:1788202474.222060:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202474.222070:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202474.222131:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559818880/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202474.222169:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202474.222174:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202474.222177:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202474.222182:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1605 00000100:00080000:1.0:1788202474.222186:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202474.222192:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202474.222204:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202474.222222:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559818880:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788202474.222223:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559819008/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202474.222248:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202474.222253:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202474.222255:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202474.222262:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1605 00000100:00080000:1.0:1788202474.222267:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202474.222271:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202474.222278:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202474.222293:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559819136/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202474.222303:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202474.222355:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202474/real 1788202474] req@ffff93c1c5037800 x1875064559818880/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202474.222371:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559818880/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202474.222380:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202474.222384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202474.222387:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559818880:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202474.222414:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559819008:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202474.222439:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202474.222453:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202474/real 1788202474] req@ffff93c1c5036680 x1875064559819008/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202474.222463:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559819008/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202474.222470:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202474.222473:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202474.222475:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559819008:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202474.222479:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559819136:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202474.222495:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202474.222502:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202474/real 1788202474] req@ffff93c1c5035c00 x1875064559819136/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202474.222510:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559819136/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202474.222517:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202474.222519:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202474.222521:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559819136:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202474.926008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202475.950056:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202476.973997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202477.997945:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202478.830030:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202478.830046:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202478.830148:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202478.830166:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202478.830169:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202478.830183:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202479.021900:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202479.341968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202479.341977:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202479.341981:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202479.341991:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1610 00000100:00080000:0.0:1788202479.342007:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202479.342013:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202479.342055:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559819264/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202479.342075:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202479.342078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202479.342079:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202479.342082:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1610 00000100:00080000:0.0:1788202479.342084:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202479.342086:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202479.342095:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202479.342104:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559819392/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788202479.342116:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559819264:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202479.342118:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202479.342123:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202479.342124:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202479.342128:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1610 00000100:00080000:0.0:1788202479.342130:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202479.342132:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202479.342136:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202479.342144:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559819520/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202479.342179:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202479.342215:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202479/real 1788202479] req@ffff93c0e9ce7480 x1875064559819264/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202479.342233:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559819264/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202479.342241:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202479.342244:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202479.342247:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559819264:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202479.342267:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559819392:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202479.342280:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202479.342287:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202479/real 1788202479] req@ffff93c0e9ce4700 x1875064559819392/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202479.342294:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559819392/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202479.342304:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202479.342306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202479.342308:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559819392:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202479.342311:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559819520:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202479.342322:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202479.342326:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202479/real 1788202479] req@ffff93c0e9ce4380 x1875064559819520/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202479.342333:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559819520/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202479.342340:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202479.342341:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202479.342343:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559819520:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202480.045960:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202481.070016:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202482.093987:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202483.117976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202483.950012:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202483.950025:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202483.950111:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202483.950130:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202483.950132:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202483.950148:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202484.141998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202484.461968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202484.461977:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202484.461980:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202484.461987:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1615 00000100:00080000:1.0:1788202484.461996:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202484.462002:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202484.462034:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559819648/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202484.462066:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202484.462070:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202484.462072:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202484.462076:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1615 00000100:00080000:1.0:1788202484.462080:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202484.462084:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202484.462093:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202484.462103:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559819776/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202484.462111:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202484.462115:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202484.462116:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788202484.462116:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559819648:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202484.462123:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1615 00000100:00080000:1.0:1788202484.462126:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202484.462128:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202484.462131:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202484.462139:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559819904/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202484.462185:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202484.462222:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202484/real 1788202484] req@ffff93c1c5035c00 x1875064559819648/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202484.462239:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559819648/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202484.462248:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202484.462252:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202484.462255:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559819648:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202484.462274:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559819776:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202484.462294:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202484.462312:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202484/real 1788202484] req@ffff93c1c5036680 x1875064559819776/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202484.462324:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559819776/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202484.462332:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202484.462335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202484.462338:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559819776:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202484.462343:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559819904:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202484.462362:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202484.462372:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202484/real 1788202484] req@ffff93c1c5037800 x1875064559819904/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202484.462382:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559819904/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202484.462392:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202484.462395:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202484.462398:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559819904:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202485.165998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202486.189980:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202487.213964:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202488.237967:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202489.069962:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202489.069980:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202489.070077:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202489.070103:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202489.070106:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202489.070126:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202489.261944:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202489.581972:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202489.581984:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202489.581988:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202489.581999:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1620 00000100:00080000:1.0:1788202489.582010:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202489.582018:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202489.582065:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559820032/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202489.582090:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202489.582095:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202489.582097:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202489.582101:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1620 00000100:00080000:1.0:1788202489.582104:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202489.582108:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202489.582117:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202489.582128:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559820160/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202489.582139:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788202489.582144:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559820032:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202489.582145:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202489.582148:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202489.582157:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1620 00000100:00080000:1.0:1788202489.582160:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202489.582164:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202489.582171:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202489.582182:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559820288/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202489.582218:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202489.582254:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202489/real 1788202489] req@ffff93c1c5037100 x1875064559820032/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202489.582268:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559820032/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202489.582275:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202489.582278:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202489.582280:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559820032:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202489.582299:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559820160:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202489.582314:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202489.582322:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202489/real 1788202489] req@ffff93c1c5034e00 x1875064559820160/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202489.582329:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559820160/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202489.582334:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202489.582336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202489.582338:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559820160:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202489.582341:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559820288:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202489.582350:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202489.582355:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202489/real 1788202489] req@ffff93c1c5034a80 x1875064559820288/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202489.582361:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559820288/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202489.582371:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202489.582373:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202489.582375:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559820288:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202490.285972:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202491.309978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202492.333944:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202493.357982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202494.190046:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202494.190064:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202494.190177:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202494.190201:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202494.190204:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202494.190222:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202494.381999:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202494.701925:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202494.701939:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202494.701943:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202494.701954:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1625 00000100:00080000:1.0:1788202494.701966:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202494.701975:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202494.702021:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559820416/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202494.702051:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202494.702054:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202494.702055:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202494.702059:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1625 00000100:00080000:1.0:1788202494.702061:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202494.702064:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202494.702073:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202494.702084:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559820544/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202494.702092:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202494.702094:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202494.702096:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202494.702098:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1625 00000100:00080000:1.0:1788202494.702100:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202494.702103:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202494.702107:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202494.702113:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559820416:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788202494.702115:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559820672/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788202494.702175:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202494.702215:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202494/real 1788202494] req@ffff93c1c5034a80 x1875064559820416/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202494.702230:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559820416/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202494.702239:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202494.702242:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202494.702247:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559820416:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202494.702253:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559820544:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788202494.702268:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202494.702276:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202494/real 1788202494] req@ffff93c1c5034e00 x1875064559820544/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202494.702284:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559820544/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202494.702290:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202494.702292:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202494.702294:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559820544:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788202494.702314:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559820672:192.168.201.156@tcp:8:kworker.0 00000800:00000100:2.0:1788202494.702325:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202494.702350:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202494/real 1788202494] req@ffff93c1c5037100 x1875064559820672/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788202494.702359:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559820672/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788202494.702365:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788202494.702367:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:2.0:1788202494.702369:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559820672:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202495.406008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202496.429972:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202497.454031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202498.477961:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202499.310024:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202499.310040:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202499.310147:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202499.310178:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202499.310183:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202499.310209:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202499.501982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202499.821930:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202499.821939:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202499.821943:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202499.821959:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1630 00000100:00080000:2.0:1788202499.821970:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202499.821976:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202499.822015:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559820800/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202499.822044:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202499.822046:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202499.822047:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202499.822050:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1630 00000100:00080000:2.0:1788202499.822052:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202499.822054:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202499.822062:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202499.822071:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559820928/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202499.822078:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202499.822081:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202499.822082:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202499.822085:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1630 00000100:00080000:2.0:1788202499.822087:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788202499.822091:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559820800:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202499.822093:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202499.822100:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202499.822108:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559821056/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202499.822162:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202499.822204:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202499/real 1788202499] req@ffff93c0e62bb480 x1875064559820800/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202499.822219:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559820800/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202499.822228:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202499.822231:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202499.822234:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559820800:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202499.822251:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559820928:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202499.822270:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202499.822281:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202499/real 1788202499] req@ffff93c0e62baa00 x1875064559820928/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202499.822294:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559820928/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202499.822303:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202499.822306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202499.822310:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559820928:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202499.822336:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559821056:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202499.822359:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202499.822373:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202499/real 1788202499] req@ffff93c0e62ba680 x1875064559821056/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202499.822385:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559821056/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202499.822395:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202499.822399:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202499.822403:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559821056:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202500.526008:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202501.550015:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202502.573974:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202503.597994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202504.430013:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202504.430028:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202504.430121:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202504.430140:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202504.430148:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202504.430185:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202504.621943:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202504.941994:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202504.942003:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202504.942008:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202504.942021:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1635 00000100:00080000:2.0:1788202504.942031:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202504.942042:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202504.942083:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559821184/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202504.942102:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202504.942105:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202504.942106:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202504.942109:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1635 00000100:00080000:2.0:1788202504.942112:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202504.942115:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202504.942122:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202504.942131:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559821184:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202504.942137:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559821312/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202504.942187:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202504.942191:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202504.942193:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202504.942198:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1635 00000100:00080000:2.0:1788202504.942202:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202504.942205:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202504.942210:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202504.942222:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559821440/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202504.942299:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559821312:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202504.942326:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559821440:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202505.645993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202506.669921:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202507.693967:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788202508.014081:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788202508.014141:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788202508.014161:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788202508.014203:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788202508.014212:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788202508.014262:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202504/real 1788202508] req@ffff93c0e62b8000 x1875064559821312/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202508.014291:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559821312/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202508.014305:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202508.014310:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202508.014315:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559821312:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202508.014324:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202504/real 1788202508] req@ffff93c0e62b8380 x1875064559821440/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202508.014338:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559821440/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202508.014348:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202508.014353:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202508.014356:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559821440:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788202508.014363:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202504/real 1788202508] req@ffff93c0e62b8700 x1875064559821184/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202508.014375:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559821184/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202508.014382:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202508.014385:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202508.014388:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559821184:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788202508.717945:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202509.549977:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202509.549992:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202509.550091:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202509.550126:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202509.550132:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202509.550157:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202509.741955:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202510.061959:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202510.061972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202510.061976:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202510.061988:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1640 00000100:00080000:1.0:1788202510.062001:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202510.062009:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202510.062048:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559821568/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202510.062098:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202510.062102:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202510.062105:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202510.062109:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1640 00000100:00080000:1.0:1788202510.062113:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202510.062118:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202510.062128:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202510.062144:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559821696/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202510.062158:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788202510.062161:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559821568:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202510.062163:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202510.062168:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202510.062175:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1640 00000100:00080000:1.0:1788202510.062179:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202510.062184:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202510.062189:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202510.062204:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559821824/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202510.062246:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202510.062304:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202510/real 1788202510] req@ffff93c1c5037800 x1875064559821568/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202510.062332:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559821568/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202510.062354:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202510.062359:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202510.062364:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559821568:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202510.062391:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559821696:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202510.062417:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202510.062432:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202510/real 1788202510] req@ffff93c1c5036680 x1875064559821696/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202510.062445:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559821696/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202510.062454:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202510.062457:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202510.062461:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559821696:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202510.062467:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559821824:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202510.062488:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202510.062501:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202510/real 1788202510] req@ffff93c1c5035c00 x1875064559821824/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202510.062513:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559821824/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202510.062530:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202510.062533:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202510.062537:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559821824:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202510.765943:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202511.789958:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202512.814157:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202513.837934:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202514.670004:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202514.670022:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202514.670111:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202514.670133:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202514.670136:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202514.670154:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202514.861964:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202515.181931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202515.181941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202515.181945:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202515.181953:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1646 00000100:00080000:1.0:1788202515.181962:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202515.181969:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202515.182022:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559821952/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202515.182052:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202515.182056:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202515.182058:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202515.182061:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1646 00000100:00080000:1.0:1788202515.182065:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202515.182068:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202515.182076:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202515.182100:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559821952:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788202515.182127:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559822080/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202515.182142:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202515.182147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202515.182149:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202515.182154:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1646 00000100:00080000:1.0:1788202515.182159:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788202515.182163:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00080000:1.0:1788202515.182167:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202515.182174:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202515.182189:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559822208/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202515.182210:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202515/real 1788202515] req@ffff93c1c5035c00 x1875064559821952/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202515.182232:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559821952/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202515.182244:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202515.182249:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202515.182253:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559821952:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202515.182275:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559822080:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202515.182300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202515.182310:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202515/real 1788202515] req@ffff93c1c5036680 x1875064559822080/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202515.182319:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559822080/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202515.182325:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202515.182328:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202515.182330:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559822080:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202515.182334:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559822208:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202515.182346:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202515.182352:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202515/real 1788202515] req@ffff93c1c5037800 x1875064559822208/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202515.182359:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559822208/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202515.182366:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202515.182368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202515.182370:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559822208:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202515.885972:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202516.909994:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202517.933961:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202518.958080:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202519.790091:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202519.790107:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202519.790231:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202519.790261:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202519.790267:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202519.790303:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202519.983138:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202520.302011:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202520.302141:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202520.302147:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202520.302157:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1651 00000100:00080000:1.0:1788202520.302169:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202520.302176:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202520.302213:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559822336/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202520.302249:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202520.302253:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202520.302255:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202520.302259:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1651 00000100:00080000:1.0:1788202520.302262:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202520.302266:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202520.302275:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202520.302288:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559822464/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202520.302298:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202520.302302:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202520.302304:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202520.302309:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1651 00000100:00080000:1.0:1788202520.302313:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202520.302317:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202520.302322:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202520.302331:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559822592/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202520.302342:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559822336:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202520.302404:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202520.302461:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202520/real 1788202520] req@ffff93c1c5037100 x1875064559822336/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202520.302483:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559822336/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202520.302495:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202520.302501:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202520.302517:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559822336:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202520.302527:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559822464:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202520.302551:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202520.302564:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202520/real 1788202520] req@ffff93c1c5034e00 x1875064559822464/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202520.302575:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559822464/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202520.302583:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202520.302587:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202520.302590:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559822464:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202520.302613:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559822592:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202520.302645:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202520.302656:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202520/real 1788202520] req@ffff93c1c5034a80 x1875064559822592/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202520.302676:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559822592/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202520.302686:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202520.302690:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202520.302693:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559822592:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202521.005962:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202522.032656:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202523.053972:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202524.077937:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202524.910023:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202524.910038:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202524.910178:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202524.910209:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202524.910214:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202524.910246:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202525.101981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202525.421956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202525.421966:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202525.421969:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202525.421977:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1656 00000100:00080000:1.0:1788202525.421987:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202525.421992:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202525.422031:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559822720/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202525.422062:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202525.422066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202525.422068:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202525.422072:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1656 00000100:00080000:1.0:1788202525.422076:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202525.422080:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202525.422091:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202525.422106:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559822848/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202525.422117:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202525.422120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202525.422122:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202525.422126:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1656 00000100:00080000:1.0:1788202525.422130:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202525.422134:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202525.422140:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202525.422154:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559822976/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202525.422173:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559822720:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202525.422243:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202525.422283:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202525/real 1788202525] req@ffff93c1c5034a80 x1875064559822720/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202525.422303:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559822720/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202525.422314:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202525.422319:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202525.422323:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559822720:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202525.422331:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559822848:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202525.422352:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202525.422372:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202525/real 1788202525] req@ffff93c1c5034e00 x1875064559822848/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202525.422387:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559822848/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202525.422396:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202525.422400:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202525.422404:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559822848:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202525.422468:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559822976:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202525.422493:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202525.422507:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202525/real 1788202525] req@ffff93c1c5037100 x1875064559822976/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202525.422520:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559822976/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202525.422542:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202525.422545:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202525.422549:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559822976:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202526.125963:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202527.150010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202528.174005:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202529.197957:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202530.030089:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202530.030105:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202530.030223:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202530.030247:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202530.030250:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202530.030273:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202530.221983:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202530.541988:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202530.541998:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202530.542002:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202530.542011:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1661 00000100:00080000:3.0:1788202530.542020:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202530.542026:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202530.542058:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559823104/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202530.542083:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202530.542086:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202530.542088:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202530.542091:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1661 00000100:00080000:3.0:1788202530.542093:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202530.542095:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202530.542102:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202530.542111:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559823232/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202530.542119:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202530.542122:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202530.542123:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202530.542126:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1661 00000100:00080000:3.0:1788202530.542128:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202530.542133:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202530.542135:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559823104:192.168.201.156@tcp:250:kworker.0 00000008:00100000:3.0:1788202530.542137:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202530.542147:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559823360/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202530.542207:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202530.542259:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202530/real 1788202530] req@ffff93c1c4a4dc00 x1875064559823104/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202530.542282:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559823104/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202530.542293:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202530.542297:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202530.542301:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559823104:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202530.542310:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559823232:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202530.542347:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202530.542362:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202530/real 1788202530] req@ffff93c1c4a4fb80 x1875064559823232/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202530.542375:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559823232/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202530.542383:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202530.542386:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202530.542390:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559823232:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202530.542414:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559823360:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202530.542436:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202530.542448:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202530/real 1788202530] req@ffff93c1c4a4e680 x1875064559823360/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202530.542461:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559823360/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202530.542476:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202530.542480:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202530.542483:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559823360:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202531.245955:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202532.270038:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202533.293981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202534.318000:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202535.150052:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202535.150069:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202535.150173:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202535.150210:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202535.150214:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202535.150483:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202535.341935:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202535.661972:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202535.661983:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202535.661988:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202535.661998:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1666 00000100:00080000:3.0:1788202535.662010:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202535.662018:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202535.662051:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559823488/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202535.662074:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202535.662077:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202535.662079:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202535.662082:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1666 00000100:00080000:3.0:1788202535.662085:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202535.662087:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202535.662094:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202535.662105:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559823616/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202535.662117:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202535.662120:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202535.662122:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202535.662129:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1666 00000100:00100000:0.0:1788202535.662132:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559823488:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202535.662134:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202535.662143:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202535.662151:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202535.662163:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559823744/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202535.662215:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202535.662266:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202535/real 1788202535] req@ffff93c1c4a4c000 x1875064559823488/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202535.662286:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559823488/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202535.662299:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202535.662304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202535.662308:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559823488:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202535.662317:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559823616:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202535.662341:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202535.662353:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202535/real 1788202535] req@ffff93c1c4a4ea00 x1875064559823616/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202535.662366:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559823616/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202535.662376:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202535.662379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202535.662383:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559823616:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202535.662410:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559823744:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202535.662431:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202535.662443:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202535/real 1788202535] req@ffff93c1c4a4c380 x1875064559823744/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202535.662457:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559823744/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202535.662468:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202535.662471:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202535.662475:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559823744:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202536.365967:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202537.390000:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202538.413954:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202539.437953:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202540.269990:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202540.270003:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202540.270098:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202540.270127:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202540.270132:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202540.270165:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202540.461979:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202540.781899:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202540.781907:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202540.781910:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202540.781918:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1671 00000100:00080000:3.0:1788202540.781928:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202540.781934:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202540.781969:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559823872/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202540.781992:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202540.781995:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202540.781996:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202540.781999:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1671 00000100:00080000:3.0:1788202540.782002:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202540.782004:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202540.782010:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202540.782019:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559824000/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202540.782027:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202540.782030:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202540.782031:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202540.782034:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1671 00000100:00080000:3.0:1788202540.782036:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202540.782038:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202540.782041:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202540.782047:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559824128/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202540.782054:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559823872:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202540.782124:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202540.782171:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202540/real 1788202540] req@ffff93c1c4a4c700 x1875064559823872/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202540.782190:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559823872/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202540.782202:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202540.782207:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202540.782211:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559823872:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202540.782220:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559824000:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202540.782242:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202540.782254:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202540/real 1788202540] req@ffff93c1c4a4f480 x1875064559824000/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202540.782267:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559824000/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202540.782275:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202540.782279:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202540.782283:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559824000:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202540.782314:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559824128:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202540.782349:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202540.782360:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202540/real 1788202540] req@ffff93c1c4a4e300 x1875064559824128/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202540.782374:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559824128/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202540.782385:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202540.782389:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202540.782393:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559824128:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202541.485936:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202542.509926:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202543.533923:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202544.557991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202545.389955:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202545.389968:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202545.390066:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202545.390095:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202545.390098:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202545.390120:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202545.581911:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202545.901912:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202545.901922:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202545.901925:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202545.901934:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1676 00000100:00080000:3.0:1788202545.901944:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202545.901950:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202545.901985:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559824256/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202545.902012:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202545.902015:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202545.902016:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202545.902028:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1676 00000100:00080000:3.0:1788202545.902031:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202545.902034:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202545.902043:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202545.902053:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559824384/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202545.902062:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202545.902067:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202545.902069:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202545.902078:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1676 00000100:00100000:0.0:1788202545.902079:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559824256:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202545.902083:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202545.902098:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202545.902105:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202545.902123:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559824512/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202545.902143:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202545.902193:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202545/real 1788202545] req@ffff93c1c4a4d880 x1875064559824256/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202545.902214:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559824256/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202545.902228:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202545.902245:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202545.902249:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559824256:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202545.902256:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559824384:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202545.902277:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202545.902304:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202545/real 1788202545] req@ffff93c1c4a4d180 x1875064559824384/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202545.902313:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559824384/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202545.902318:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202545.902321:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202545.902323:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559824384:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202545.902346:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559824512:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202545.902367:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202545.902380:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202545/real 1788202545] req@ffff93c1c4a4f800 x1875064559824512/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202545.902392:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559824512/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202545.902399:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202545.902401:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202545.902403:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559824512:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202546.605940:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202547.629979:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202548.653955:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202549.677930:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202550.509992:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202550.510006:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202550.510087:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202550.510106:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202550.510109:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202550.510127:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202550.701996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202551.021907:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202551.021915:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202551.021918:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202551.021926:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1681 00000100:00080000:3.0:1788202551.021936:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202551.021941:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202551.021972:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ca80 x1875064559824640/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202551.021994:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202551.021997:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202551.021998:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202551.022001:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1681 00000100:00080000:3.0:1788202551.022003:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202551.022005:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202551.022012:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202551.022021:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d500 x1875064559824768/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202551.022029:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202551.022033:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788202551.022035:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559824640:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202551.022035:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202551.022040:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1681 00000100:00080000:3.0:1788202551.022046:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202551.022048:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202551.022052:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202551.022060:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559824896/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202551.022110:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202551.022165:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202551/real 1788202551] req@ffff93c1c4a4ca80 x1875064559824640/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202551.022187:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ca80 x1875064559824640/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202551.022199:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202551.022203:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202551.022208:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559824640:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202551.022232:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559824768:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202551.022259:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202551.022274:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202551/real 1788202551] req@ffff93c1c4a4d500 x1875064559824768/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202551.022288:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d500 x1875064559824768/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202551.022298:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202551.022301:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202551.022305:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559824768:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202551.022312:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559824896:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202551.022334:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202551.022347:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202551/real 1788202551] req@ffff93c1c4a4f800 x1875064559824896/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202551.022376:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559824896/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202551.022387:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202551.022391:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202551.022395:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559824896:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202551.725952:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202552.749948:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202553.773974:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202554.798013:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202555.630069:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202555.630092:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202555.630204:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202555.630230:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202555.630233:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202555.630250:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202555.822026:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202556.141923:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202556.141930:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202556.141934:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202556.141942:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1687 00000100:00080000:3.0:1788202556.141951:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202556.141957:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202556.141989:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559825024/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202556.142011:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202556.142014:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202556.142015:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202556.142018:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1687 00000100:00080000:3.0:1788202556.142021:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202556.142023:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202556.142030:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202556.142038:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559825152/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202556.142045:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202556.142048:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202556.142049:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202556.142052:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1687 00000100:00080000:3.0:1788202556.142054:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202556.142057:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202556.142059:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202556.142067:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559825280/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202556.142078:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559825024:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202556.142145:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202556.142203:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202556/real 1788202556] req@ffff93c1c4a4d180 x1875064559825024/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202556.142229:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559825024/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202556.142241:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202556.142245:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202556.142250:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559825024:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202556.142259:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559825152:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202556.142305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202556.142320:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202556/real 1788202556] req@ffff93c1c4a4d880 x1875064559825152/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202556.142334:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559825152/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202556.142342:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202556.142347:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202556.142350:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559825152:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202556.142375:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559825280:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202556.142396:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202556.142409:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202556/real 1788202556] req@ffff93c1c4a4e300 x1875064559825280/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202556.142423:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559825280/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202556.142433:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202556.142437:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202556.142440:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559825280:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202556.846033:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202557.870028:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202558.893985:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202559.917985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202560.750001:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202560.750017:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202560.750113:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202560.750136:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202560.750139:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202560.750158:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202560.941957:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202561.261942:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202561.261951:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202561.261954:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202561.261962:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1692 00000100:00080000:3.0:1788202561.261971:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202561.261977:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202561.262010:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559825408/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202561.262031:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202561.262034:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202561.262035:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202561.262039:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1692 00000100:00080000:3.0:1788202561.262041:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202561.262043:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202561.262051:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202561.262060:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559825536/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202561.262078:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202561.262082:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202561.262084:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788202561.262090:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559825408:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202561.262091:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1692 00000100:00080000:3.0:1788202561.262099:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202561.262105:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202561.262111:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202561.262124:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559825664/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202561.262161:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202561.262216:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202561/real 1788202561] req@ffff93c1c4a4f480 x1875064559825408/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202561.262234:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559825408/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202561.262245:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202561.262249:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202561.262253:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559825408:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202561.262261:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559825536:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202561.262295:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202561.262308:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202561/real 1788202561] req@ffff93c1c4a4c700 x1875064559825536/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202561.262322:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559825536/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202561.262330:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202561.262334:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202561.262337:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559825536:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202561.262360:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559825664:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202561.262381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202561.262392:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202561/real 1788202561] req@ffff93c1c4a4c380 x1875064559825664/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202561.262406:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559825664/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202561.262417:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202561.262420:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202561.262424:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559825664:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202561.966031:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202562.989976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202564.013975:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202565.037974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202565.869991:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202565.870004:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202565.870111:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202565.870134:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202565.870137:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202565.870159:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202566.061954:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202566.381934:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202566.381943:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202566.381947:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202566.381955:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1697 00000100:00080000:3.0:1788202566.381965:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202566.381970:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202566.382001:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559825792/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202566.382026:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202566.382029:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202566.382031:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202566.382035:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1697 00000100:00080000:3.0:1788202566.382039:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202566.382043:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202566.382052:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202566.382065:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559825920/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202566.382075:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202566.382078:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202566.382080:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788202566.382082:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559825792:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202566.382097:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1697 00000100:00080000:3.0:1788202566.382101:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202566.382105:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202566.382110:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202566.382120:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559826048/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202566.382165:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202566.382217:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202566/real 1788202566] req@ffff93c1c4a4ea00 x1875064559825792/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202566.382237:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559825792/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202566.382249:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202566.382254:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202566.382259:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559825792:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202566.382287:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559825920:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202566.382307:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202566.382316:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202566/real 1788202566] req@ffff93c1c4a4c000 x1875064559825920/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202566.382325:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559825920/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202566.382331:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202566.382333:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202566.382335:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559825920:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202566.382339:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559826048:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202566.382350:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202566.382356:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202566/real 1788202566] req@ffff93c1c4a4e680 x1875064559826048/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202566.382364:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559826048/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202566.382375:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202566.382377:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202566.382380:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559826048:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202567.085932:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202568.109966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202569.134067:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202570.157909:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202570.989958:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202570.989974:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202570.990075:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202570.990096:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202570.990100:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202570.990125:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202571.181966:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202571.501986:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202571.502006:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202571.502009:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202571.502018:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1702 00000100:00080000:1.0:1788202571.502027:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202571.502033:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202571.502066:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559826176/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202571.502091:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202571.502093:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202571.502095:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202571.502103:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1702 00000100:00080000:1.0:1788202571.502106:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202571.502109:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202571.502117:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202571.502125:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559826304/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202571.502134:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788202571.502139:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559826176:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202571.502139:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202571.502141:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202571.502149:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1702 00000100:00080000:1.0:1788202571.502151:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202571.502153:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202571.502157:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202571.502167:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559826432/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202571.502240:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559826304:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202571.502267:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559826432:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202572.205951:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202573.229996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202574.253931:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788202574.574173:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788202574.574225:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788202574.574248:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202574.574301:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202574.574315:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788202574.574374:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202571/real 1788202574] req@ffff93c1c5036680 x1875064559826304/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202574.574416:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559826304/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202574.574429:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202574.574435:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202574.574440:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559826304:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202574.574451:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202571/real 1788202574] req@ffff93c1c5035c00 x1875064559826432/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202574.574466:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559826432/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202574.574478:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202574.574483:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202574.574486:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559826432:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788202574.574495:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202571/real 1788202574] req@ffff93c1c5037800 x1875064559826176/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202574.574522:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559826176/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202574.574531:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202574.574535:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202574.574540:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559826176:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788202575.278017:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202576.110069:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202576.110082:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202576.110169:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202576.110188:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202576.110191:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202576.110209:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202576.302039:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202576.622004:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202576.622015:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202576.622019:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202576.622031:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1707 00000100:00080000:2.0:1788202576.622043:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202576.622051:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202576.622091:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559826560/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202576.622117:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202576.622120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202576.622121:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202576.622125:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1707 00000100:00080000:2.0:1788202576.622127:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202576.622130:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202576.622139:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202576.622149:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559826688/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202576.622157:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202576.622160:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202576.622161:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202576.622164:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1707 00000100:00080000:2.0:1788202576.622167:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202576.622171:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202576.622176:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202576.622181:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559826560:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202576.622201:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559826816/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202576.622272:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202576.622336:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202576/real 1788202576] req@ffff93c0e62bb100 x1875064559826560/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202576.622358:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559826560/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202576.622371:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202576.622377:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202576.622381:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559826560:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202576.622391:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559826688:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202576.622422:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202576.622439:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202576/real 1788202576] req@ffff93c0e62bbb80 x1875064559826688/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202576.622452:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559826688/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202576.622462:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202576.622465:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202576.622469:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559826688:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202576.622501:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559826816:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202576.622533:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202576.622546:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202576/real 1788202576] req@ffff93c0e62b9c00 x1875064559826816/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202576.622559:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559826816/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202576.622571:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202576.622574:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202576.622578:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559826816:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202577.326025:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202578.349958:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202579.373991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202580.397937:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202581.230055:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202581.230069:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202581.230151:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202581.230171:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202581.230174:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202581.230192:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202581.421980:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202581.741950:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202581.741958:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202581.741961:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202581.741969:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1712 00000100:00080000:1.0:1788202581.741978:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202581.741984:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202581.742018:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559826944/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202581.742041:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202581.742044:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202581.742046:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202581.742049:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1712 00000100:00080000:1.0:1788202581.742051:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202581.742053:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202581.742061:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202581.742068:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559827072/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202581.742075:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202581.742077:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202581.742079:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202581.742081:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1712 00000100:00080000:1.0:1788202581.742083:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202581.742086:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202581.742089:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202581.742096:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559827200/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202581.742109:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559826944:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202581.742176:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202581.742224:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202581/real 1788202581] req@ffff93c1c5037800 x1875064559826944/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202581.742247:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559826944/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202581.742259:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202581.742264:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202581.742269:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559826944:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202581.742277:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559827072:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202581.742300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202581.742312:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202581/real 1788202581] req@ffff93c1c5035c00 x1875064559827072/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202581.742325:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559827072/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202581.742334:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202581.742338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202581.742348:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559827072:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202581.742371:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559827200:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202581.742390:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202581.742399:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202581/real 1788202581] req@ffff93c1c5036680 x1875064559827200/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202581.742412:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559827200/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202581.742423:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202581.742426:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202581.742429:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559827200:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202582.445987:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202583.469988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202584.493967:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202585.517974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202586.350010:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202586.350025:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202586.350127:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202586.350152:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202586.350156:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202586.350184:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202586.541978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202586.861965:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202586.861974:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202586.861977:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202586.861986:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1717 00000100:00080000:2.0:1788202586.861995:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202586.862001:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202586.862035:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559827328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202586.862064:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202586.862069:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202586.862071:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202586.862076:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1717 00000100:00080000:2.0:1788202586.862079:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202586.862084:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202586.862094:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202586.862108:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559827456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202586.862112:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559827328:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202586.862122:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202586.862126:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202586.862127:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202586.862132:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1717 00000100:00080000:2.0:1788202586.862135:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202586.862138:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202586.862142:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202586.862152:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559827584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202586.862183:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202586.862241:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202586/real 1788202586] req@ffff93c0e62bad80 x1875064559827328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202586.862261:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559827328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202586.862270:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202586.862273:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202586.862276:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559827328:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202586.862295:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559827456:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202586.862315:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202586.862325:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202586/real 1788202586] req@ffff93c0e62b8a80 x1875064559827456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202586.862333:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559827456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202586.862338:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202586.862341:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202586.862343:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559827456:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202586.862346:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559827584:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202586.862357:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202586.862363:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202586/real 1788202586] req@ffff93c0e62b9500 x1875064559827584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202586.862370:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559827584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202586.862377:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202586.862378:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202586.862380:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559827584:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202587.565967:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202588.589966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202589.613946:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202590.638009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202591.470029:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202591.470042:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202591.470124:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202591.470142:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202591.470145:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202591.470161:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202591.662040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202591.981960:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202591.981969:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202591.981973:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202591.981981:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1722 00000100:00080000:2.0:1788202591.981991:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202591.981997:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202591.982033:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559827712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202591.982063:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202591.982065:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202591.982067:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202591.982071:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1722 00000100:00080000:2.0:1788202591.982074:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202591.982078:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202591.982087:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202591.982096:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559827840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202591.982103:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202591.982106:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202591.982107:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202591.982109:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1722 00000100:00080000:2.0:1788202591.982111:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202591.982114:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202591.982118:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202591.982128:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559827968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202591.982128:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559827712:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788202591.982196:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202591.982236:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202591/real 1788202591] req@ffff93c0e62b9f80 x1875064559827712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202591.982254:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559827712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202591.982265:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202591.982269:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202591.982274:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559827712:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202591.982281:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559827840:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202591.982304:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202591.982318:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202591/real 1788202591] req@ffff93c0e62bb800 x1875064559827840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202591.982331:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559827840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202591.982340:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202591.982344:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202591.982347:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559827840:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202591.982373:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559827968:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202591.982393:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202591.982405:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202591/real 1788202591] req@ffff93c0e62b9500 x1875064559827968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202591.982417:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559827968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202591.982427:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202591.982431:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202591.982434:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559827968:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202592.686034:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202593.709929:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202594.733924:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202595.757995:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202596.590027:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202596.590042:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202596.590140:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202596.590167:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202596.590172:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202596.590202:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202596.782058:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202597.101949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202597.101959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202597.101963:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202597.101971:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1727 00000100:00080000:2.0:1788202597.101980:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202597.101986:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202597.102019:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559828096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202597.102045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202597.102048:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202597.102049:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202597.102053:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1727 00000100:00080000:2.0:1788202597.102055:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202597.102058:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202597.102065:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202597.102074:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559828224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202597.102082:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202597.102085:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202597.102087:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202597.102090:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1727 00000100:00080000:2.0:1788202597.102093:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202597.102095:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202597.102098:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202597.102102:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559828096:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202597.102108:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559828352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202597.102173:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202597.102208:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202597/real 1788202597] req@ffff93c0e62b8a80 x1875064559828096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202597.102222:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559828096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202597.102230:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202597.102233:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202597.102236:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559828096:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202597.102242:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559828224:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202597.102258:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202597.102266:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202597/real 1788202597] req@ffff93c0e62bad80 x1875064559828224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202597.102274:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559828224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202597.102280:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202597.102282:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202597.102284:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559828224:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202597.102302:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559828352:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202597.102314:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202597.102319:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202597/real 1788202597] req@ffff93c0e62b9c00 x1875064559828352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202597.102327:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559828352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202597.102333:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202597.102335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202597.102337:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559828352:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202597.806038:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202598.830047:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202599.853980:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202600.878080:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202601.710069:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202601.710083:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202601.710167:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202601.710201:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202601.710207:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202601.710239:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202601.902025:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202602.221965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202602.221974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202602.221977:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202602.221985:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1733 00000100:00080000:3.0:1788202602.221995:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202602.222001:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202602.222033:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559828480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202602.222071:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202602.222075:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202602.222077:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202602.222082:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1733 00000100:00080000:3.0:1788202602.222086:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202602.222089:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202602.222098:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202602.222113:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559828608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202602.222125:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559828480:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202602.222127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202602.222135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202602.222137:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202602.222143:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1733 00000100:00080000:3.0:1788202602.222149:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202602.222153:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202602.222160:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202602.222173:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559828736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202602.222187:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202602.222230:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202602/real 1788202602] req@ffff93c1c4a4fb80 x1875064559828480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202602.222244:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559828480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202602.222251:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202602.222254:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202602.222257:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559828480:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202602.222275:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559828608:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202602.222291:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202602.222299:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202602/real 1788202602] req@ffff93c1c4a4dc00 x1875064559828608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202602.222307:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559828608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202602.222313:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202602.222315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202602.222317:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559828608:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202602.222321:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559828736:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202602.222332:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202602.222338:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202602/real 1788202602] req@ffff93c1c4a4ed80 x1875064559828736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202602.222345:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559828736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202602.222352:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202602.222355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202602.222358:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559828736:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202602.925987:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202603.950041:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202604.974036:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202605.998073:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202606.830032:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202606.830045:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202606.830152:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202606.830190:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202606.830196:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202606.830258:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202607.021952:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202607.342006:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202607.342021:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202607.342026:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202607.342039:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1738 00000100:00080000:1.0:1788202607.342054:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202607.342062:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202607.342112:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559828864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202607.342149:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202607.342153:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202607.342155:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202607.342161:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1738 00000100:00080000:1.0:1788202607.342165:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202607.342169:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202607.342181:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202607.342195:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559828992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202607.342204:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559828864:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202607.342212:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202607.342217:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202607.342219:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202607.342225:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1738 00000100:00080000:1.0:1788202607.342231:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202607.342235:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202607.342241:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202607.342254:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559829120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202607.342281:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202607.342324:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202607/real 1788202607] req@ffff93c1c5037100 x1875064559828864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202607.342340:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559828864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202607.342348:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202607.342352:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202607.342355:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559828864:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202607.342401:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559828992:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202607.342421:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202607.342432:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202607/real 1788202607] req@ffff93c1c5034e00 x1875064559828992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202607.342440:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559828992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202607.342446:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202607.342449:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202607.342451:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559828992:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202607.342455:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559829120:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202607.342468:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202607.342474:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202607/real 1788202607] req@ffff93c1c5034a80 x1875064559829120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202607.342482:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559829120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202607.342490:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202607.342492:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202607.342494:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559829120:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202608.046011:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202609.070045:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202610.094051:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202611.117980:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202611.950078:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202611.950106:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202611.950244:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202611.950287:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202611.950324:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202611.950367:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202612.142022:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202612.461966:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202612.461993:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202612.461999:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202612.462012:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1743 00000100:00080000:0.0:1788202612.462029:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202612.462039:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202612.462102:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559829248/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202612.462140:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202612.462145:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202612.462147:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202612.462153:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1743 00000100:00080000:0.0:1788202612.462157:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202612.462162:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202612.462173:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202612.462196:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559829248:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202612.462203:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559829376/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202612.462218:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202612.462223:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202612.462225:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202612.462229:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1743 00000100:00080000:0.0:1788202612.462233:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202612.462237:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202612.462242:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202612.462253:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559829504/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202612.462300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202612.462349:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202612/real 1788202612] req@ffff93c0e9ce6a00 x1875064559829248/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202612.462378:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559829248/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202612.462389:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202612.462407:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202612.462412:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559829248:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202612.462443:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559829376:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202612.462468:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202612.462481:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202612/real 1788202612] req@ffff93c0e9ce4000 x1875064559829376/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202612.462494:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559829376/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202612.462503:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202612.462507:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202612.462511:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559829376:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202612.462518:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559829504:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202612.462540:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202612.462551:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202612/real 1788202612] req@ffff93c0e9ce6680 x1875064559829504/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202612.462563:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559829504/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202612.462580:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202612.462584:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202612.462587:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559829504:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202613.166015:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202614.190068:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202615.214012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202616.238075:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202617.070013:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202617.070037:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202617.070164:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202617.070197:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202617.070201:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202617.070240:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202617.261991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202617.582011:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202617.582024:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202617.582029:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202617.582043:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1748 00000100:00080000:3.0:1788202617.582057:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202617.582067:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202617.582118:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4df80 x1875064559829632/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202617.582153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202617.582157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202617.582159:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202617.582166:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1748 00000100:00080000:3.0:1788202617.582170:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202617.582176:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202617.582188:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202617.582193:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559829632:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202617.582220:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f100 x1875064559829760/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202617.582235:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202617.582239:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202617.582241:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202617.582247:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1748 00000100:00080000:3.0:1788202617.582251:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202617.582256:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202617.582263:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202617.582288:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559829888/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202617.582297:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202617.582354:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202617/real 1788202617] req@ffff93c1c4a4df80 x1875064559829632/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202617.582370:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4df80 x1875064559829632/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202617.582380:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202617.582383:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202617.582387:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559829632:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202617.582412:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559829760:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202617.582440:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202617.582454:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202617/real 1788202617] req@ffff93c1c4a4f100 x1875064559829760/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202617.582469:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f100 x1875064559829760/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202617.582479:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202617.582483:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202617.582486:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559829760:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202617.582497:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559829888:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202617.582520:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202617.582533:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202617/real 1788202617] req@ffff93c1c4a4ed80 x1875064559829888/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202617.582547:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559829888/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202617.582559:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202617.582561:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202617.582564:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559829888:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202618.286002:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202619.309921:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202620.334025:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202621.357921:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202622.190001:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202622.190018:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202622.190145:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202622.190177:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202622.190182:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202622.190209:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202622.382035:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202622.701966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202622.701982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202622.701987:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202622.702000:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1753 00000100:00080000:3.0:1788202622.702011:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202622.702019:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202622.702064:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559830016/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202622.702092:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202622.702095:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202622.702096:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202622.702100:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1753 00000100:00080000:3.0:1788202622.702102:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202622.702117:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202622.702130:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202622.702143:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559830144/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202622.702150:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559830016:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202622.702154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202622.702158:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202622.702160:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202622.702164:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1753 00000100:00080000:3.0:1788202622.702167:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202622.702170:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202622.702174:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202622.702185:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559830272/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202622.702229:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202622.702278:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202622/real 1788202622] req@ffff93c1c4a4dc00 x1875064559830016/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202622.702295:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559830016/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202622.702304:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202622.702308:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202622.702312:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559830016:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202622.702334:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559830144:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202622.702353:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202622.702361:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202622/real 1788202622] req@ffff93c1c4a4fb80 x1875064559830144/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202622.702370:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559830144/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202622.702376:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202622.702378:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202622.702380:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559830144:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202622.702384:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559830272:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202622.702395:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202622.702400:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202622/real 1788202622] req@ffff93c1c4a4e680 x1875064559830272/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202622.702408:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559830272/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202622.702415:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202622.702418:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202622.702420:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559830272:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202623.405933:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202624.430116:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202625.453978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202626.478090:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202627.309958:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202627.309970:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202627.310086:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202627.310121:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202627.310125:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202627.310149:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202627.501912:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202627.821924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202627.821933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202627.821936:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202627.821944:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1758 00000100:00080000:3.0:1788202627.821961:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202627.821970:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202627.822003:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559830400/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202627.822044:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202627.822049:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202627.822051:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202627.822056:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1758 00000100:00080000:3.0:1788202627.822059:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202627.822064:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202627.822074:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202627.822088:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559830528/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202627.822099:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559830400:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202627.822100:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202627.822108:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202627.822111:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202627.822115:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1758 00000100:00080000:3.0:1788202627.822117:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202627.822120:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202627.822124:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202627.822133:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559830656/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202627.822177:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202627.822219:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202627/real 1788202627] req@ffff93c1c4a4c000 x1875064559830400/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202627.822239:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559830400/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202627.822250:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202627.822261:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202627.822266:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559830400:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202627.822289:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559830528:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202627.822309:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202627.822319:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202627/real 1788202627] req@ffff93c1c4a4ea00 x1875064559830528/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202627.822328:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559830528/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202627.822334:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202627.822336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202627.822338:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559830528:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202627.822342:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559830656:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202627.822353:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202627.822359:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202627/real 1788202627] req@ffff93c1c4a4c380 x1875064559830656/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202627.822367:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559830656/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202627.822378:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202627.822380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202627.822382:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559830656:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202628.526030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202629.549968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202630.574067:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202631.598029:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202632.430047:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202632.430064:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202632.430185:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202632.430217:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202632.430234:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202632.430280:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202632.622058:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202632.941965:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202632.941975:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202632.941979:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202632.941989:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1763 00000100:00080000:3.0:1788202632.941999:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202632.942006:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202632.942051:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559830784/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202632.942080:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202632.942084:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202632.942086:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202632.942096:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1763 00000100:00080000:3.0:1788202632.942099:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202632.942103:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202632.942112:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202632.942124:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559830912/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202632.942135:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202632.942139:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202632.942143:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788202632.942145:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559830784:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202632.942150:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1763 00000100:00080000:3.0:1788202632.942165:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202632.942170:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202632.942175:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202632.942188:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559831040/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202632.942221:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202632.942265:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202632/real 1788202632] req@ffff93c1c4a4c700 x1875064559830784/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202632.942284:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559830784/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202632.942292:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202632.942294:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202632.942297:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559830784:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202632.942316:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559830912:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202632.942334:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202632.942343:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202632/real 1788202632] req@ffff93c1c4a4f480 x1875064559830912/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202632.942355:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559830912/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202632.942363:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202632.942367:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202632.942370:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559830912:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202632.942377:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559831040:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202632.942397:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202632.942409:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202632/real 1788202632] req@ffff93c1c4a4e300 x1875064559831040/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202632.942422:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559831040/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202632.942432:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202632.942436:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202632.942440:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559831040:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202633.646025:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202634.669986:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202635.694013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202636.718017:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202637.550023:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202637.550040:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202637.550176:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202637.550207:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202637.550212:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202637.550244:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202637.742056:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202638.061916:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202638.061933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202638.061938:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202638.061949:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1768 00000100:00080000:3.0:1788202638.061961:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202638.061969:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202638.062020:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559831168/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202638.062056:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202638.062060:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202638.062062:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202638.062067:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1768 00000100:00080000:3.0:1788202638.062071:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202638.062076:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202638.062089:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202638.062108:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559831296/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202638.062115:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559831168:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202638.062128:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202638.062133:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202638.062135:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202638.062141:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1768 00000100:00080000:3.0:1788202638.062145:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202638.062150:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202638.062156:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202638.062170:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559831424/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202638.062246:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559831296:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202638.062278:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559831424:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202638.766019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202639.790011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202640.813983:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788202641.134203:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788202641.134255:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788202641.134284:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202641.134343:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202641.134357:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788202641.134392:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202638/real 1788202641] req@ffff93c1c4a4d180 x1875064559831296/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202641.134416:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559831296/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202641.134426:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202641.134430:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202641.134434:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559831296:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202641.134443:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202638/real 1788202641] req@ffff93c1c4a4f800 x1875064559831424/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202641.134456:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559831424/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202641.134468:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202641.134472:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202641.134476:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559831424:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788202641.134484:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202638/real 1788202641] req@ffff93c1c4a4d880 x1875064559831168/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202641.134496:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559831168/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202641.134505:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202641.134509:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202641.134513:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559831168:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788202641.838018:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202642.669949:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202642.669962:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202642.670050:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202642.670072:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202642.670075:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202642.670093:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202642.862019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202643.181967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202643.181976:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202643.181980:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202643.181992:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1774 00000100:00080000:0.0:1788202643.182004:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202643.182013:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202643.182054:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559831552/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202643.182084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202643.182087:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202643.182089:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202643.182094:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1774 00000100:00080000:0.0:1788202643.182098:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202643.182103:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202643.182113:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202643.182123:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559831552:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202643.182133:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559831680/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202643.182151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202643.182154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202643.182156:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202643.182160:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1774 00000100:00080000:0.0:1788202643.182164:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202643.182167:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202643.182173:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202643.182187:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559831808/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202643.182207:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202643.182252:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202643/real 1788202643] req@ffff93c0e9ce7b80 x1875064559831552/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202643.182273:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559831552/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202643.182283:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202643.182287:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202643.182291:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559831552:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202643.182318:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559831680:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202643.182343:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202643.182357:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202643/real 1788202643] req@ffff93c0e9ce5c00 x1875064559831680/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202643.182369:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559831680/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202643.182376:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202643.182379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202643.182383:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559831680:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202643.182392:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559831808:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202643.182413:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202643.182425:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202643/real 1788202643] req@ffff93c0e9ce6d80 x1875064559831808/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202643.182436:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559831808/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202643.182467:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202643.182470:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202643.182474:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559831808:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202643.886039:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202644.910022:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202645.933958:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202646.958044:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202647.789974:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202647.789987:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202647.790069:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202647.790089:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202647.790092:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202647.790109:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202647.981917:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202648.301978:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202648.301989:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202648.301994:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202648.302004:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1779 00000100:00080000:3.0:1788202648.302015:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202648.302023:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202648.302058:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d500 x1875064559831936/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202648.302090:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202648.302094:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202648.302096:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202648.302102:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1779 00000100:00080000:3.0:1788202648.302107:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202648.302112:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202648.302121:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202648.302125:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559831936:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202648.302135:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ca80 x1875064559832064/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202648.302145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202648.302148:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202648.302149:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202648.302152:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1779 00000100:00080000:3.0:1788202648.302155:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202648.302157:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202648.302161:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202648.302170:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559832192/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202648.302188:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202648.302230:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202648/real 1788202648] req@ffff93c1c4a4d500 x1875064559831936/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202648.302245:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d500 x1875064559831936/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202648.302254:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202648.302257:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202648.302261:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559831936:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202648.302283:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559832064:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202648.302300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202648.302309:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202648/real 1788202648] req@ffff93c1c4a4ca80 x1875064559832064/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202648.302317:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ca80 x1875064559832064/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202648.302322:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202648.302325:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202648.302327:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559832064:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202648.302331:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559832192:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202648.302341:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202648.302347:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202648/real 1788202648] req@ffff93c1c4a4d880 x1875064559832192/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202648.302354:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559832192/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202648.302361:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202648.302363:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202648.302365:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559832192:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202649.006014:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202650.029939:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202651.054038:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202652.077923:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202652.909999:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202652.910012:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202652.910092:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202652.910112:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202652.910115:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202652.910132:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202653.101988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202653.421970:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202653.421979:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202653.421983:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202653.421991:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1784 00000100:00080000:1.0:1788202653.422001:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202653.422006:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202653.422043:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559832320/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202653.422069:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202653.422072:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202653.422073:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202653.422076:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1784 00000100:00080000:1.0:1788202653.422078:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202653.422081:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202653.422089:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202653.422098:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559832448/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202653.422107:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202653.422110:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202653.422111:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202653.422114:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1784 00000100:00080000:1.0:1788202653.422116:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202653.422121:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202653.422122:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559832320:192.168.201.156@tcp:250:kworker.0 00000008:00100000:1.0:1788202653.422124:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202653.422135:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559832576/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202653.422194:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202653.422234:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202653/real 1788202653] req@ffff93c1c5034a80 x1875064559832320/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202653.422250:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559832320/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202653.422263:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202653.422266:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202653.422269:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559832320:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202653.422278:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559832448:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202653.422293:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202653.422301:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202653/real 1788202653] req@ffff93c1c5034e00 x1875064559832448/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202653.422309:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559832448/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202653.422325:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202653.422327:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202653.422329:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559832448:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202653.422346:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559832576:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202653.422357:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202653.422363:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202653/real 1788202653] req@ffff93c1c5037100 x1875064559832576/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202653.422371:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559832576/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202653.422382:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202653.422384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202653.422386:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559832576:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202654.125952:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202655.150004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202656.173955:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202657.198056:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202658.029990:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202658.030002:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202658.030096:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202658.030117:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202658.030119:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202658.030137:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202658.221962:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202658.541931:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202658.541940:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202658.541944:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202658.541953:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1789 00000100:00080000:1.0:1788202658.541963:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202658.541971:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202658.542008:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559832704/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202658.542031:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202658.542033:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202658.542035:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202658.542038:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1789 00000100:00080000:1.0:1788202658.542040:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202658.542043:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202658.542050:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202658.542060:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559832832/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202658.542067:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202658.542070:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202658.542071:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202658.542074:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1789 00000100:00080000:1.0:1788202658.542075:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202658.542078:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202658.542082:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202658.542084:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559832704:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788202658.542091:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559832960/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202658.542148:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202658.542191:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202658/real 1788202658] req@ffff93c1c5036680 x1875064559832704/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202658.542208:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559832704/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202658.542216:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202658.542219:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202658.542222:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559832704:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202658.542227:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559832832:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202658.542242:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202658.542250:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202658/real 1788202658] req@ffff93c1c5035c00 x1875064559832832/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202658.542258:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559832832/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202658.542268:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202658.542270:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202658.542273:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559832832:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202658.542290:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559832960:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202658.542302:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202658.542307:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202658/real 1788202658] req@ffff93c1c5037800 x1875064559832960/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202658.542315:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559832960/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202658.542322:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202658.542324:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202658.542326:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559832960:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202659.246010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202660.269988:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202661.294053:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202662.317932:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202663.150062:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202663.150080:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202663.150210:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202663.150235:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202663.150239:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202663.150260:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202663.342085:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202663.662009:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202663.662024:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202663.662029:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202663.662042:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1794 00000100:00080000:3.0:1788202663.662056:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202663.662065:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202663.662118:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559833088/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202663.662152:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202663.662164:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202663.662166:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202663.662172:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1794 00000100:00080000:3.0:1788202663.662177:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202663.662183:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202663.662196:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202663.662205:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559833088:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202663.662211:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559833216/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202663.662226:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202663.662229:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202663.662231:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202663.662234:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1794 00000100:00080000:3.0:1788202663.662237:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202663.662240:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202663.662243:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202663.662254:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559833344/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202663.662295:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202663.662359:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202663/real 1788202663] req@ffff93c1c4a4f800 x1875064559833088/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202663.662384:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559833088/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202663.662418:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202663.662424:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202663.662429:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559833088:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202663.662456:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559833216:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202663.662477:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202663.662487:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202663/real 1788202663] req@ffff93c1c4a4d180 x1875064559833216/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202663.662496:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559833216/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202663.662505:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202663.662508:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202663.662512:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559833216:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202663.662519:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559833344:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202663.662541:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202663.662553:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202663/real 1788202663] req@ffff93c1c4a4e300 x1875064559833344/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202663.662565:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559833344/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202663.662573:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202663.662575:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202663.662577:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559833344:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202664.365981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202665.390048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202666.414020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202667.437998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202668.269936:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202668.269953:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202668.270044:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202668.270066:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202668.270069:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202668.270088:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202668.461954:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202668.781947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202668.781957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202668.781961:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202668.781970:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1799 00000100:00080000:3.0:1788202668.781982:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202668.781990:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202668.782028:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559833472/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202668.782060:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202668.782064:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202668.782066:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202668.782070:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1799 00000100:00080000:3.0:1788202668.782074:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202668.782078:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202668.782088:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202668.782102:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559833600/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202668.782115:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788202668.782120:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559833472:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202668.782120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202668.782124:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202668.782133:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1799 00000100:00080000:3.0:1788202668.782137:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202668.782141:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202668.782147:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202668.782160:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559833728/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202668.782190:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202668.782236:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202668/real 1788202668] req@ffff93c1c4a4f480 x1875064559833472/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202668.782257:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559833472/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202668.782268:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202668.782273:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202668.782277:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559833472:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202668.782300:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559833600:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202668.782342:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202668.782355:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202668/real 1788202668] req@ffff93c1c4a4c700 x1875064559833600/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202668.782369:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559833600/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202668.782378:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202668.782382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202668.782386:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559833600:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202668.782393:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559833728:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202668.782413:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202668.782425:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202668/real 1788202668] req@ffff93c1c4a4c380 x1875064559833728/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202668.782438:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559833728/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202668.782448:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202668.782452:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202668.782456:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559833728:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202669.485987:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202670.509963:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202671.534012:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202672.557949:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202673.390063:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202673.390080:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202673.390191:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202673.390224:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202673.390246:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202673.390283:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202673.581985:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202673.901981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202673.901993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202673.901998:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202673.902009:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1804 00000100:00080000:3.0:1788202673.902023:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202673.902031:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202673.902077:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559833856/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202673.902107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202673.902110:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202673.902111:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202673.902115:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1804 00000100:00080000:3.0:1788202673.902117:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202673.902120:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202673.902130:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202673.902139:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559833984/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202673.902147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202673.902150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202673.902151:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202673.902156:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1804 00000100:00080000:3.0:1788202673.902158:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202673.902161:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202673.902165:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202673.902172:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559834112/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202673.902243:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559833856:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202673.902326:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202673.902385:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202673/real 1788202673] req@ffff93c1c4a4ea00 x1875064559833856/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202673.902417:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559833856/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202673.902431:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202673.902436:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202673.902440:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559833856:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202673.902451:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559833984:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202673.902477:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202673.902491:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202673/real 1788202673] req@ffff93c1c4a4c000 x1875064559833984/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202673.902512:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559833984/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202673.902521:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202673.902524:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202673.902528:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559833984:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202673.902535:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559834112:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202673.902560:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202673.902573:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202673/real 1788202673] req@ffff93c1c4a4e680 x1875064559834112/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202673.902586:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559834112/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202673.902598:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202673.902602:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202673.902606:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559834112:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202674.605957:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202675.629963:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202676.654028:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202677.677957:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202678.510031:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202678.510052:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202678.510216:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202678.510263:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202678.510269:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202678.510305:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202678.702015:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202679.022918:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202679.022929:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202679.022933:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202679.022943:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1809 00000100:00080000:2.0:1788202679.022954:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202679.022960:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202679.022992:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559834240/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202679.023012:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202679.023015:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202679.023016:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202679.023019:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1809 00000100:00080000:2.0:1788202679.023022:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202679.023026:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202679.023035:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202679.023040:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559834240:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202679.023047:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559834368/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202679.023055:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202679.023057:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202679.023058:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202679.023061:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1809 00000100:00080000:2.0:1788202679.023063:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202679.023066:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202679.023069:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202679.023078:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559834496/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202679.023096:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202679.023146:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202679/real 1788202679] req@ffff93c0e62bbb80 x1875064559834240/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202679.023165:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559834240/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202679.023173:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202679.023176:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202679.023178:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559834240:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202679.023197:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559834368:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202679.023213:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202679.023221:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202679/real 1788202679] req@ffff93c0e62bb100 x1875064559834368/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202679.023229:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559834368/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202679.023234:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202679.023236:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202679.023238:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559834368:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202679.023242:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559834496:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202679.023253:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202679.023258:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202679/real 1788202679] req@ffff93c0e62b8700 x1875064559834496/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202679.023266:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559834496/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202679.023276:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202679.023278:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202679.023280:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559834496:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202679.725963:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202680.750005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202681.773953:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202682.797955:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202683.629971:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202683.629989:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202683.630126:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202683.630153:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202683.630158:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202683.630190:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202683.822027:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202684.141933:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202684.141942:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202684.141946:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202684.141954:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1815 00000100:00080000:2.0:1788202684.141963:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202684.141970:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202684.142001:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559834624/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202684.142027:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202684.142030:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202684.142032:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202684.142036:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1815 00000100:00080000:2.0:1788202684.142050:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202684.142055:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202684.142065:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202684.142079:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559834752/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202684.142092:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202684.142096:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202684.142098:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788202684.142103:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559834624:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202684.142106:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1815 00000100:00080000:2.0:1788202684.142113:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202684.142116:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202684.142121:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202684.142132:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559834880/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202684.142165:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202684.142205:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202684/real 1788202684] req@ffff93c0e62b8380 x1875064559834624/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202684.142220:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559834624/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202684.142229:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202684.142232:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202684.142235:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559834624:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202684.142240:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559834752:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202684.142255:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202684.142262:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202684/real 1788202684] req@ffff93c0e62b8000 x1875064559834752/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202684.142271:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559834752/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202684.142276:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202684.142278:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202684.142281:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559834752:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202684.142299:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559834880:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202684.142311:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202684.142317:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202684/real 1788202684] req@ffff93c0e62ba680 x1875064559834880/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202684.142324:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559834880/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202684.142331:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202684.142332:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202684.142334:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559834880:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202684.845897:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202685.870060:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202686.894018:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202687.918090:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202688.750107:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202688.750129:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202688.750280:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202688.750317:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202688.750323:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202688.750363:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202688.942040:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202689.262030:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202689.262044:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202689.262049:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202689.262062:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1820 00000100:00080000:3.0:1788202689.262074:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202689.262084:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202689.262131:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559835008/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202689.262164:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202689.262168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202689.262170:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202689.262175:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1820 00000100:00080000:3.0:1788202689.262178:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202689.262183:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202689.262194:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202689.262208:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559835136/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202689.262221:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788202689.262228:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559835008:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202689.262228:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202689.262231:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202689.262240:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1820 00000100:00080000:3.0:1788202689.262244:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202689.262247:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202689.262252:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202689.262265:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559835264/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202689.262323:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202689.262388:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202689/real 1788202689] req@ffff93c1c4a4fb80 x1875064559835008/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202689.262411:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559835008/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202689.262423:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202689.262428:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202689.262432:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559835008:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202689.262457:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559835136:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202689.262478:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202689.262488:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202689/real 1788202689] req@ffff93c1c4a4dc00 x1875064559835136/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202689.262497:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559835136/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202689.262502:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202689.262505:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202689.262507:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559835136:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202689.262510:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559835264:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202689.262522:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202689.262528:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202689/real 1788202689] req@ffff93c1c4a4ed80 x1875064559835264/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202689.262535:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559835264/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202689.262542:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202689.262544:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202689.262546:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559835264:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202689.966044:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202690.989993:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202692.014039:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202693.037975:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202693.870030:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202693.870043:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202693.870127:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202693.870153:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202693.870157:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202693.870182:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202694.062041:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202694.381955:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202694.381963:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202694.381967:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202694.381976:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1825 00000100:00080000:1.0:1788202694.381987:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202694.381994:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202694.382036:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559835392/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202694.382060:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202694.382063:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202694.382068:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202694.382072:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1825 00000100:00080000:1.0:1788202694.382075:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202694.382078:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202694.382086:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202694.382097:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559835520/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202694.382111:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788202694.382115:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559835392:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202694.382115:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202694.382118:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202694.382127:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1825 00000100:00080000:1.0:1788202694.382130:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202694.382132:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202694.382136:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202694.382147:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559835648/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202694.382183:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202694.382232:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202694/real 1788202694] req@ffff93c1c5037800 x1875064559835392/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202694.382247:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559835392/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202694.382257:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202694.382260:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202694.382263:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559835392:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202694.382282:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559835520:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202694.382298:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202694.382306:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202694/real 1788202694] req@ffff93c1c5035c00 x1875064559835520/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202694.382314:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559835520/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202694.382320:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202694.382322:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202694.382324:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559835520:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202694.382328:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559835648:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202694.382341:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202694.382348:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202694/real 1788202694] req@ffff93c1c5036680 x1875064559835648/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202694.382361:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559835648/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202694.382368:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202694.382370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202694.382371:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559835648:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202695.085989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202696.110007:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202697.133951:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202698.158074:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202698.990014:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202698.990031:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202698.990142:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202698.990167:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202698.990170:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202698.990187:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202699.181968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202699.502019:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202699.502032:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202699.502037:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202699.502050:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1830 00000100:00080000:3.0:1788202699.502069:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202699.502080:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202699.502135:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f100 x1875064559835776/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202699.502169:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202699.502173:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202699.502176:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202699.502181:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1830 00000100:00080000:3.0:1788202699.502185:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202699.502190:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202699.502203:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202699.502218:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4df80 x1875064559835904/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202699.502222:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559835776:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202699.502239:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202699.502244:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202699.502247:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202699.502253:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1830 00000100:00080000:3.0:1788202699.502257:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202699.502261:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202699.502266:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202699.502279:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559836032/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202699.502301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202699.502350:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202699/real 1788202699] req@ffff93c1c4a4f100 x1875064559835776/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202699.502366:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f100 x1875064559835776/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202699.502379:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202699.502384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202699.502389:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559835776:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202699.502428:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559835904:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202699.502448:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202699.502458:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202699/real 1788202699] req@ffff93c1c4a4df80 x1875064559835904/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202699.502482:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4df80 x1875064559835904/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202699.502501:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202699.502504:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202699.502508:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559835904:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202699.502516:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559836032:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202699.502548:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202699.502561:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202699/real 1788202699] req@ffff93c1c4a4ed80 x1875064559836032/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202699.502574:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559836032/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202699.502586:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202699.502590:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202699.502593:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559836032:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202700.206008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202701.229903:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202702.254105:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202703.277980:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202704.109963:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202704.109976:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202704.110068:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202704.110095:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202704.110098:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202704.110117:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202704.302013:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202704.621949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202704.621961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202704.621965:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202704.621977:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1835 00000100:00080000:3.0:1788202704.621989:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202704.621997:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202704.622041:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559836160/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202704.622063:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202704.622066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202704.622067:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202704.622071:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1835 00000100:00080000:3.0:1788202704.622074:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202704.622076:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202704.622088:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202704.622097:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559836288/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202704.622104:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202704.622106:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202704.622108:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202704.622111:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1835 00000100:00080000:3.0:1788202704.622113:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202704.622116:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202704.622120:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202704.622129:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559836416/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202704.622130:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559836160:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202704.622254:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559836288:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202704.622309:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559836416:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202705.325939:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202706.350022:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202707.374010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788202707.694099:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788202707.694142:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788202707.694157:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202707.694202:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202707.694213:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788202707.694261:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202704/real 1788202707] req@ffff93c1c4a4e680 x1875064559836416/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202707.694287:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559836416/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202707.694300:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202707.694305:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202707.694309:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559836416:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788202707.694318:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202704/real 1788202707] req@ffff93c1c4a4dc00 x1875064559836160/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202707.694331:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559836160/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202707.694339:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202707.694343:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202707.694347:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559836160:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202707.694354:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202704/real 1788202707] req@ffff93c1c4a4fb80 x1875064559836288/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202707.694366:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559836288/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202707.694374:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202707.694377:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202707.694381:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559836288:192.168.201.156@tcp:38:kworker.0 00010000:00010000:3.0:1788202708.398018:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202709.230010:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202709.230024:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202709.230101:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202709.230118:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202709.230121:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202709.230136:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202709.421955:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202709.741944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202709.741956:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202709.741960:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202709.741971:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1840 00000100:00080000:0.0:1788202709.741981:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202709.741988:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202709.742021:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559836544/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202709.742050:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202709.742054:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202709.742055:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202709.742059:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1840 00000100:00080000:0.0:1788202709.742062:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202709.742066:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202709.742075:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202709.742088:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559836672/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202709.742091:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559836544:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202709.742110:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202709.742113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202709.742115:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202709.742119:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1840 00000100:00080000:0.0:1788202709.742122:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202709.742126:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202709.742130:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202709.742141:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559836800/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202709.742169:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202709.742221:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202709/real 1788202709] req@ffff93c0e9ce7100 x1875064559836544/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202709.742240:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559836544/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202709.742250:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202709.742254:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202709.742258:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559836544:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202709.742284:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559836672:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202709.742304:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202709.742316:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202709/real 1788202709] req@ffff93c0e9ce5f80 x1875064559836672/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202709.742328:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559836672/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202709.742334:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202709.742337:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202709.742339:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559836672:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202709.742344:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559836800:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202709.742361:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202709.742369:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202709/real 1788202709] req@ffff93c0e9ce4a80 x1875064559836800/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202709.742379:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559836800/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202709.742388:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202709.742390:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202709.742393:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559836800:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202710.445975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202711.469966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202712.493978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202713.517960:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202714.350036:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202714.350053:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202714.350168:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202714.350189:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202714.350192:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202714.350209:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202714.542020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202714.861972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202714.861982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202714.861987:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202714.861998:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1845 00000100:00080000:0.0:1788202714.862010:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202714.862018:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202714.862064:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559836928/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202714.862086:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202714.862088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202714.862089:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202714.862093:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1845 00000100:00080000:0.0:1788202714.862094:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202714.862097:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202714.862105:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202714.862113:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559837056/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202714.862120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202714.862123:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202714.862124:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202714.862129:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1845 00000100:00100000:3.0:1788202714.862132:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559836928:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202714.862133:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202714.862138:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202714.862143:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202714.862150:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559837184/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202714.862206:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202714.862257:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202714/real 1788202714] req@ffff93c0e9ce7800 x1875064559836928/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202714.862271:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559836928/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202714.862279:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202714.862283:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202714.862285:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559836928:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202714.862311:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559837056:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202714.862326:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202714.862334:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202714/real 1788202714] req@ffff93c0e9ce5180 x1875064559837056/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202714.862346:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559837056/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202714.862351:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202714.862353:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202714.862355:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559837056:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202714.862358:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559837184:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202714.862369:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202714.862374:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202714/real 1788202714] req@ffff93c0e9ce5500 x1875064559837184/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202714.862380:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559837184/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202714.862387:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202714.862388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202714.862390:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559837184:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202715.565985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202716.589957:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202717.613979:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202718.637967:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202719.469983:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202719.470001:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202719.470099:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202719.470120:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202719.470124:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202719.470143:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202719.661978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202719.981962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202719.981973:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202719.981978:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202719.981989:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1850 00000100:00080000:3.0:1788202719.982001:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202719.982009:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202719.982056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559837312/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202719.982085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202719.982089:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202719.982091:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202719.982095:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1850 00000100:00080000:3.0:1788202719.982099:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202719.982104:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202719.982114:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202719.982121:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559837312:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202719.982132:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559837440/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202719.982145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202719.982149:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202719.982152:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202719.982157:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1850 00000100:00080000:3.0:1788202719.982161:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202719.982165:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202719.982171:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000800:00000100:0.0:1788202719.982179:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202719.982185:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559837568/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202719.982220:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202719/real 1788202719] req@ffff93c1c4a4c000 x1875064559837312/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202719.982235:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559837312/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202719.982243:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202719.982246:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202719.982249:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559837312:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202719.982265:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559837440:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202719.982280:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202719.982288:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202719/real 1788202719] req@ffff93c1c4a4ea00 x1875064559837440/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202719.982295:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559837440/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202719.982301:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202719.982303:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202719.982306:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559837440:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202719.982311:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559837568:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202719.982322:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202719.982329:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202719/real 1788202719] req@ffff93c1c4a4c380 x1875064559837568/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202719.982336:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559837568/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202719.982343:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202719.982345:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202719.982346:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559837568:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202720.685998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202721.709963:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202722.734008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202723.757964:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202724.589979:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202724.589994:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202724.590092:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202724.590120:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202724.590125:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202724.590155:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202724.781988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202725.101961:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202725.101969:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202725.101972:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202725.101981:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1855 00000100:00080000:0.0:1788202725.101989:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202725.101996:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202725.102027:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559837696/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202725.102050:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202725.102052:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202725.102054:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202725.102057:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1855 00000100:00080000:0.0:1788202725.102060:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202725.102063:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202725.102071:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202725.102081:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559837824/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202725.102087:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559837696:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202725.102093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202725.102096:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202725.102097:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202725.102100:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1855 00000100:00080000:0.0:1788202725.102102:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202725.102105:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202725.102109:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202725.102116:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559837952/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202725.102146:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202725.102178:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202725/real 1788202725] req@ffff93c0e9ce4e00 x1875064559837696/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202725.102190:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559837696/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202725.102198:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202725.102201:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202725.102204:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559837696:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202725.102223:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559837824:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202725.102237:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202725.102244:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202725/real 1788202725] req@ffff93c0e9ce5880 x1875064559837824/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202725.102252:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559837824/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202725.102257:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202725.102260:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202725.102262:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559837824:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202725.102266:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559837952:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202725.102276:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202725.102281:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202725/real 1788202725] req@ffff93c0e9ce6300 x1875064559837952/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202725.102293:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559837952/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202725.102299:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202725.102301:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202725.102303:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559837952:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202725.806023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202726.830072:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202727.854044:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202728.878015:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202729.709997:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202729.710016:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202729.710099:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202729.710115:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202729.710118:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202729.710136:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202729.902040:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202730.221991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202730.222008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202730.222014:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202730.222027:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1861 00000100:00080000:2.0:1788202730.222042:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202730.222052:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202730.222106:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559838080/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202730.222136:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202730.222139:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202730.222141:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202730.222144:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1861 00000100:00080000:2.0:1788202730.222147:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202730.222151:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202730.222161:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202730.222171:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559838208/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202730.222180:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202730.222183:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202730.222185:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202730.222190:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1861 00000100:00080000:2.0:1788202730.222192:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202730.222195:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202730.222199:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202730.222207:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559838336/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202730.222234:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559838080:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788202730.222355:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202730.222401:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202730/real 1788202730] req@ffff93c0e62baa00 x1875064559838080/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202730.222421:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559838080/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202730.222432:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202730.222436:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202730.222441:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559838080:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202730.222450:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559838208:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202730.222474:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202730.222488:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202730/real 1788202730] req@ffff93c0e62bb480 x1875064559838208/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202730.222500:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559838208/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202730.222509:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202730.222512:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202730.222515:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559838208:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202730.222541:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559838336:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202730.222560:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202730.222570:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202730/real 1788202730] req@ffff93c0e62ba680 x1875064559838336/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202730.222582:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559838336/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202730.222592:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202730.222595:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202730.222599:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559838336:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202730.926044:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202731.950035:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202732.974066:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202733.997992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202734.830009:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202734.830023:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202734.830116:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202734.830138:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202734.830142:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202734.830187:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202735.021950:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202735.341913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202735.341925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202735.341929:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202735.341940:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1866 00000100:00080000:0.0:1788202735.341953:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202735.341961:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202735.342007:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559838464/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202735.342033:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202735.342036:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202735.342037:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202735.342041:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1866 00000100:00080000:0.0:1788202735.342044:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202735.342047:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202735.342057:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202735.342068:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559838464:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202735.342071:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559838592/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202735.342085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202735.342089:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202735.342091:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202735.342096:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1866 00000100:00080000:0.0:1788202735.342100:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202735.342105:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202735.342111:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202735.342126:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559838720/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202735.342146:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202735.342213:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202735/real 1788202735] req@ffff93c0e9ce6300 x1875064559838464/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202735.342230:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559838464/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202735.342243:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202735.342246:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202735.342250:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559838464:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202735.342274:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559838592:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202735.342291:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202735.342299:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202735/real 1788202735] req@ffff93c0e9ce5880 x1875064559838592/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202735.342308:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559838592/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202735.342313:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202735.342316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202735.342318:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559838592:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202735.342322:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559838720:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202735.342332:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202735.342338:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202735/real 1788202735] req@ffff93c0e9ce4e00 x1875064559838720/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202735.342345:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559838720/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202735.342353:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202735.342355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202735.342356:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559838720:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202736.045956:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202737.070095:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202738.093944:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202739.118002:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202739.950017:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202739.950029:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202739.950134:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202739.950153:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202739.950157:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202739.950180:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202740.141960:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202740.461979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202740.461987:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202740.461990:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202740.461999:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1871 00000100:00080000:0.0:1788202740.462008:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202740.462015:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202740.462049:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559838848/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202740.462099:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202740.462102:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202740.462104:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202740.462107:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1871 00000100:00080000:0.0:1788202740.462110:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202740.462113:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202740.462123:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202740.462134:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559838976/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202740.462143:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202740.462146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202740.462148:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788202740.462152:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559838848:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202740.462154:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1871 00000100:00080000:0.0:1788202740.462160:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202740.462164:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202740.462167:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202740.462176:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559839104/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202740.462215:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202740.462254:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202740/real 1788202740] req@ffff93c0e9ce5500 x1875064559838848/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202740.462267:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559838848/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202740.462275:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202740.462278:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202740.462282:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559838848:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202740.462306:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559838976:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202740.462321:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202740.462330:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202740/real 1788202740] req@ffff93c0e9ce5180 x1875064559838976/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202740.462337:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559838976/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202740.462343:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202740.462345:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202740.462347:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559838976:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202740.462351:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559839104:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202740.462363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202740.462369:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202740/real 1788202740] req@ffff93c0e9ce7800 x1875064559839104/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202740.462377:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559839104/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202740.462392:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202740.462394:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202740.462396:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559839104:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202741.166065:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202742.190002:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202743.214002:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202744.237959:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202745.070011:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202745.070027:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202745.070127:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202745.070151:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202745.070155:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202745.070182:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202745.262015:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202745.581942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202745.581951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202745.581955:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202745.581964:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1876 00000100:00080000:0.0:1788202745.581987:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202745.581995:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202745.582037:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559839232/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202745.582069:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202745.582072:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202745.582074:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202745.582077:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1876 00000100:00080000:0.0:1788202745.582080:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202745.582083:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202745.582092:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202745.582103:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559839360/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202745.582112:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202745.582115:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202745.582117:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202745.582120:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1876 00000100:00080000:0.0:1788202745.582122:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202745.582126:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202745.582128:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559839232:192.168.201.156@tcp:250:kworker.0 00000008:00100000:0.0:1788202745.582130:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202745.582141:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559839488/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202745.582193:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202745.582236:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202745/real 1788202745] req@ffff93c0e9ce4a80 x1875064559839232/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202745.582250:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559839232/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202745.582259:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202745.582262:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202745.582265:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559839232:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202745.582287:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559839360:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202745.582305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202745.582318:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202745/real 1788202745] req@ffff93c0e9ce5f80 x1875064559839360/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202745.582332:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559839360/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202745.582342:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202745.582345:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202745.582349:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559839360:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202745.582356:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559839488:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202745.582374:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202745.582382:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202745/real 1788202745] req@ffff93c0e9ce7100 x1875064559839488/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202745.582390:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559839488/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202745.582397:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202745.582399:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202745.582401:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559839488:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202746.286035:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202747.309995:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202748.333985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202749.358023:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202750.190082:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202750.190104:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202750.190253:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202750.190302:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202750.190308:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202750.190353:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202750.381968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202750.701924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202750.701932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202750.701935:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202750.701943:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1881 00000100:00080000:0.0:1788202750.701954:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202750.701962:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202750.702002:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559839616/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202750.702030:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202750.702035:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202750.702037:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202750.702042:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1881 00000100:00080000:0.0:1788202750.702046:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202750.702050:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202750.702066:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202750.702083:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559839744/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202750.702083:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559839616:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202750.702102:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202750.702106:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202750.702109:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202750.702113:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1881 00000100:00080000:0.0:1788202750.702117:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202750.702121:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202750.702127:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202750.702139:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559839872/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202750.702155:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202750.702212:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202750/real 1788202750] req@ffff93c0e9ce6d80 x1875064559839616/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202750.702240:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559839616/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202750.702252:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202750.702258:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202750.702262:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559839616:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202750.702286:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559839744:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202750.702329:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202750.702343:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202750/real 1788202750] req@ffff93c0e9ce5c00 x1875064559839744/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202750.702357:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559839744/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202750.702366:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202750.702369:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202750.702373:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559839744:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202750.702380:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559839872:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202750.702400:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202750.702411:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202750/real 1788202750] req@ffff93c0e9ce7b80 x1875064559839872/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202750.702424:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559839872/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202750.702441:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202750.702444:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202750.702447:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559839872:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202751.405967:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202752.429965:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202753.453992:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202754.477982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202755.309986:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202755.310000:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202755.310110:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202755.310139:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202755.310142:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202755.310163:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202755.502053:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202755.821989:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202755.822000:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202755.822005:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202755.822016:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1886 00000100:00080000:1.0:1788202755.822029:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202755.822038:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202755.822094:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559840000/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202755.822126:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202755.822131:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202755.822133:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202755.822138:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1886 00000100:00080000:1.0:1788202755.822141:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202755.822146:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202755.822155:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202755.822168:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559840128/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202755.822178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788202755.822179:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559840000:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202755.822184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202755.822188:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202755.822193:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1886 00000100:00080000:1.0:1788202755.822195:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202755.822197:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202755.822201:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202755.822210:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559840256/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202755.822262:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202755.822337:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202755/real 1788202755] req@ffff93c1c5037100 x1875064559840000/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202755.822363:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559840000/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202755.822376:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202755.822381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202755.822386:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559840000:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202755.822416:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559840128:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202755.822443:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202755.822458:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202755/real 1788202755] req@ffff93c1c5034e00 x1875064559840128/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202755.822472:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559840128/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202755.822487:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202755.822490:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202755.822492:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559840128:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202755.822498:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559840256:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202755.822516:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202755.822525:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202755/real 1788202755] req@ffff93c1c5034a80 x1875064559840256/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202755.822533:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559840256/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202755.822541:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202755.822543:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202755.822546:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559840256:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202756.526039:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202757.550013:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202758.573965:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202759.598069:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202760.430059:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202760.430081:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202760.430207:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202760.430235:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202760.430239:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202760.430261:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202760.622011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202760.941988:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202760.941999:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202760.942005:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202760.942016:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1891 00000100:00080000:2.0:1788202760.942029:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202760.942036:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202760.942081:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559840384/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202760.942107:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202760.942110:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202760.942112:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202760.942115:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1891 00000100:00080000:2.0:1788202760.942118:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202760.942120:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202760.942130:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202760.942140:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559840512/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202760.942147:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202760.942151:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202760.942152:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202760.942155:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1891 00000100:00080000:2.0:1788202760.942157:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202760.942160:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202760.942166:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202760.942166:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559840384:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202760.942175:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559840640/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202760.942255:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202760.942308:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202760/real 1788202760] req@ffff93c0e62b8000 x1875064559840384/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202760.942325:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559840384/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202760.942333:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202760.942338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202760.942341:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559840384:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202760.942347:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559840512:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202760.942364:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202760.942372:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202760/real 1788202760] req@ffff93c0e62b8380 x1875064559840512/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202760.942381:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559840512/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202760.942387:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202760.942389:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202760.942391:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559840512:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202760.942423:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559840640:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202760.942453:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202760.942466:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202760/real 1788202760] req@ffff93c0e62b8700 x1875064559840640/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202760.942480:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559840640/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202760.942492:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202760.942494:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202760.942497:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559840640:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788202761.646074:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202762.669954:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202763.694086:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202764.717977:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202765.550027:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202765.550045:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202765.550155:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202765.550176:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202765.550179:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202765.550197:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788202765.742081:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202766.061983:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202766.061995:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202766.062000:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202766.062011:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1896 00000100:00080000:2.0:1788202766.062025:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202766.062035:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202766.062089:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559840768/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202766.062113:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202766.062116:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202766.062118:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202766.062122:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1896 00000100:00080000:2.0:1788202766.062125:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202766.062130:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202766.062140:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202766.062153:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559840896/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202766.062155:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559840768:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202766.062173:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202766.062177:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202766.062180:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202766.062186:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1896 00000100:00080000:2.0:1788202766.062190:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202766.062194:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202766.062201:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202766.062226:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559841024/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202766.062240:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202766.062290:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202766/real 1788202766] req@ffff93c0e62bb100 x1875064559840768/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202766.062307:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559840768/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202766.062317:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202766.062320:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202766.062323:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559840768:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202766.062346:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559840896:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202766.062363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202766.062371:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202766/real 1788202766] req@ffff93c0e62bbb80 x1875064559840896/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202766.062379:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559840896/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202766.062385:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202766.062388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202766.062390:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559840896:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202766.062394:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559841024:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202766.062405:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202766.062412:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202766/real 1788202766] req@ffff93c0e62b9c00 x1875064559841024/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202766.062420:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559841024/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202766.062427:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202766.062429:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202766.062431:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559841024:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202766.766031:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202767.790057:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202768.813955:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788202769.838061:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202770.670032:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202770.670051:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202770.670157:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202770.670179:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202770.670182:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202770.670199:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202770.862010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202771.181957:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202771.181970:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202771.181974:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202771.181986:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1902 00000100:00080000:2.0:1788202771.182000:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202771.182008:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202771.182058:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559841152/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202771.182093:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202771.182097:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202771.182099:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202771.182103:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1902 00000100:00080000:2.0:1788202771.182108:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202771.182112:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202771.182132:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202771.182150:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559841280/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202771.182164:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202771.182168:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202771.182170:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202771.182177:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1902 00000100:00080000:2.0:1788202771.182184:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202771.182184:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559841152:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202771.182200:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202771.182208:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202771.182224:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559841408/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202771.182301:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559841280:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202771.182333:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559841408:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202771.886068:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202772.909996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202773.933979:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788202774.254075:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:0.0:1788202774.254117:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788202774.254133:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788202774.254162:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788202774.254168:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788202774.254205:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202771/real 1788202774] req@ffff93c0e62b9500 x1875064559841408/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202774.254233:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559841408/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202774.254255:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202774.254260:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202774.254263:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559841408:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788202774.254270:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202771/real 1788202774] req@ffff93c0e62bad80 x1875064559841152/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202774.254286:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559841152/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202774.254295:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202774.254299:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202774.254302:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559841152:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202774.254309:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202771/real 1788202774] req@ffff93c0e62b8a80 x1875064559841280/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202774.254322:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559841280/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202774.254331:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202774.254335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202774.254338:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559841280:192.168.201.156@tcp:38:kworker.0 00010000:00010000:0.0:1788202774.957933:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202775.789964:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202775.789978:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202775.790096:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202775.790116:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202775.790120:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202775.790141:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202775.982006:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202776.301943:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202776.301952:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202776.301959:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202776.301968:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1907 00000100:00080000:2.0:1788202776.301978:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202776.301984:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202776.302020:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559841536/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202776.302053:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202776.302056:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202776.302058:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202776.302063:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1907 00000100:00080000:2.0:1788202776.302066:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202776.302071:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202776.302081:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202776.302094:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559841664/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202776.302104:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202776.302108:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202776.302110:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202776.302114:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1907 00000100:00080000:2.0:1788202776.302117:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202776.302124:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202776.302125:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559841536:192.168.201.156@tcp:250:kworker.0 00000008:00100000:2.0:1788202776.302129:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202776.302143:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559841792/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202776.302196:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202776.302248:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202776/real 1788202776] req@ffff93c0e62bb800 x1875064559841536/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202776.302268:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559841536/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202776.302298:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202776.302302:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202776.302307:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559841536:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202776.302317:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559841664:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202776.302341:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202776.302356:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202776/real 1788202776] req@ffff93c0e62b9f80 x1875064559841664/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202776.302370:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559841664/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202776.302379:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202776.302382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202776.302386:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559841664:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202776.302414:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559841792:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202776.302437:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202776.302464:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202776/real 1788202776] req@ffff93c0e62b8a80 x1875064559841792/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202776.302483:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559841792/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202776.302496:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202776.302499:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202776.302503:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559841792:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202777.005910:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202778.030046:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202779.053988:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202780.077999:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202780.910068:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202780.910090:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202780.910231:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202780.910265:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202780.910270:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202780.910303:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202781.101914:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202781.421948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202781.421956:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202781.421960:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202781.421968:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1912 00000100:00080000:0.0:1788202781.421982:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202781.421987:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202781.422021:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559841920/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202781.422042:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202781.422045:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202781.422047:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202781.422050:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1912 00000100:00080000:0.0:1788202781.422052:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202781.422055:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202781.422064:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202781.422073:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559842048/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202781.422081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202781.422085:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202781.422087:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788202781.422091:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559841920:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202781.422095:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1912 00000100:00080000:0.0:1788202781.422101:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202781.422105:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202781.422112:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202781.422127:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559842176/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202781.422153:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202781.422185:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202781/real 1788202781] req@ffff93c0e9ce6680 x1875064559841920/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202781.422199:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559841920/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202781.422207:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202781.422210:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202781.422213:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559841920:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202781.422232:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559842048:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202781.422246:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202781.422254:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202781/real 1788202781] req@ffff93c0e9ce4000 x1875064559842048/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202781.422262:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559842048/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202781.422268:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202781.422270:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202781.422273:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559842048:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202781.422279:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559842176:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202781.422303:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202781.422316:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202781/real 1788202781] req@ffff93c0e9ce6a00 x1875064559842176/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202781.422330:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559842176/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202781.422345:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202781.422347:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202781.422377:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559842176:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202782.125974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202783.149926:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202784.173935:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202785.197996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202786.029962:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202786.029975:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202786.030093:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202786.030130:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202786.030137:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202786.030175:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202786.221931:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202786.541975:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202786.541983:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202786.541987:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202786.541995:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1917 00000100:00080000:3.0:1788202786.542004:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202786.542010:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202786.542042:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559842304/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202786.542064:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202786.542067:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202786.542068:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202786.542071:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1917 00000100:00080000:3.0:1788202786.542073:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202786.542076:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202786.542083:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202786.542092:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559842432/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202786.542099:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202786.542102:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202786.542104:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202786.542107:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1917 00000100:00080000:3.0:1788202786.542109:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788202786.542114:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559842304:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202786.542114:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202786.542119:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202786.542129:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559842560/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202786.542181:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202786.542234:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202786/real 1788202786] req@ffff93c1c4a4c700 x1875064559842304/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202786.542249:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559842304/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202786.542259:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202786.542288:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202786.542293:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559842304:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202786.542301:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559842432:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202786.542327:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202786.542341:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202786/real 1788202786] req@ffff93c1c4a4f480 x1875064559842432/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202786.542352:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559842432/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202786.542358:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202786.542361:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202786.542363:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559842432:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202786.542384:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559842560:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202786.542399:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202786.542406:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202786/real 1788202786] req@ffff93c1c4a4e300 x1875064559842560/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202786.542415:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559842560/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202786.542440:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202786.542444:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202786.542447:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559842560:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202787.245970:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202788.269952:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202789.294010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202790.317921:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202791.149993:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202791.150007:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202791.150089:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202791.150110:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202791.150112:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202791.150129:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202791.342002:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202791.661893:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202791.661900:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202791.661904:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202791.661913:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1922 00000100:00080000:3.0:1788202791.661924:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202791.661932:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202791.661972:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559842688/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202791.661996:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202791.661999:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202791.662000:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202791.662004:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1922 00000100:00080000:3.0:1788202791.662007:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202791.662010:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202791.662018:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202791.662027:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559842816/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202791.662035:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202791.662037:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202791.662039:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202791.662042:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1922 00000100:00080000:3.0:1788202791.662044:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202791.662047:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202791.662050:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202791.662056:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559842944/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202791.662065:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559842688:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202791.662132:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202791.662215:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202791/real 1788202791] req@ffff93c1c4a4d180 x1875064559842688/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202791.662237:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559842688/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202791.662249:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202791.662254:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202791.662258:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559842688:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202791.662267:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559842816:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202791.662292:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202791.662306:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202791/real 1788202791] req@ffff93c1c4a4f800 x1875064559842816/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202791.662320:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559842816/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202791.662329:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202791.662333:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202791.662337:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559842816:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202791.662361:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559842944:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202791.662383:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202791.662396:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202791/real 1788202791] req@ffff93c1c4a4d880 x1875064559842944/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202791.662408:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559842944/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202791.662419:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202791.662423:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202791.662426:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559842944:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202792.365956:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202793.389982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202794.413929:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202795.437952:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202796.270012:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202796.270034:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202796.270182:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202796.270209:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202796.270213:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202796.270233:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202796.461981:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202796.781955:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202796.781966:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202796.781971:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202796.781982:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1927 00000100:00080000:2.0:1788202796.781994:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202796.782002:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202796.782046:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559843072/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202796.782072:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202796.782076:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202796.782078:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202796.782097:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1927 00000100:00080000:2.0:1788202796.782101:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202796.782107:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202796.782120:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202796.782137:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559843200/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202796.782147:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202796.782150:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202796.782152:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202796.782156:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1927 00000100:00080000:2.0:1788202796.782160:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788202796.782167:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559843072:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202796.782169:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202796.782183:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202796.782216:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559843328/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202796.782246:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202796.782303:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202796/real 1788202796] req@ffff93c0e62bad80 x1875064559843072/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202796.782324:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559843072/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202796.782334:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202796.782338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202796.782342:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559843072:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202796.782349:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559843200:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202796.782370:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202796.782379:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202796/real 1788202796] req@ffff93c0e62b9500 x1875064559843200/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202796.782388:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559843200/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202796.782394:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202796.782405:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202796.782408:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559843200:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202796.782429:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559843328:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202796.782447:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202796.782455:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202796/real 1788202796] req@ffff93c0e62b9c00 x1875064559843328/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202796.782463:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559843328/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202796.782471:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202796.782473:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202796.782475:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559843328:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202797.486003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202798.510023:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202799.533966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202800.558032:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202801.390067:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202801.390087:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202801.390226:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202801.390250:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202801.390253:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202801.390273:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202801.582020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202801.901968:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202801.901983:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202801.901988:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202801.902036:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1932 00000100:00080000:2.0:1788202801.902049:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202801.902058:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202801.902104:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559843456/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202801.902133:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202801.902136:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202801.902137:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202801.902141:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1932 00000100:00080000:2.0:1788202801.902144:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202801.902147:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202801.902157:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202801.902169:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559843584/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202801.902178:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202801.902181:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202801.902182:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788202801.902188:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559843456:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202801.902188:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1932 00000100:00080000:2.0:1788202801.902190:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202801.902197:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202801.902201:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202801.902209:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559843712/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202801.902276:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202801.902329:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202801/real 1788202801] req@ffff93c0e62bbb80 x1875064559843456/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202801.902352:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559843456/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202801.902365:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202801.902370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202801.902375:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559843456:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202801.902404:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559843584:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202801.902425:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202801.902435:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202801/real 1788202801] req@ffff93c0e62bb100 x1875064559843584/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202801.902444:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559843584/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202801.902450:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202801.902453:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202801.902455:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559843584:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202801.902460:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559843712:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202801.902472:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202801.902479:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202801/real 1788202801] req@ffff93c0e62b8700 x1875064559843712/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202801.902486:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559843712/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202801.902494:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202801.902496:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202801.902498:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559843712:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202802.606056:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202803.629958:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202804.654043:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202805.678035:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202806.510090:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202806.510111:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202806.510255:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202806.510284:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202806.510289:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202806.510319:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202806.701985:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202807.021952:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202807.021965:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202807.021970:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202807.021982:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1937 00000100:00080000:2.0:1788202807.022007:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202807.022018:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202807.022069:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559843840/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202807.022100:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202807.022103:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202807.022105:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202807.022108:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1937 00000100:00080000:2.0:1788202807.022111:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202807.022115:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202807.022125:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202807.022136:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559843968/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202807.022144:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202807.022149:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788202807.022151:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559843840:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202807.022151:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202807.022158:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1937 00000100:00080000:2.0:1788202807.022161:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202807.022163:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202807.022167:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202807.022176:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559844096/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202807.022232:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202807.022272:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202807/real 1788202807] req@ffff93c0e62b8380 x1875064559843840/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202807.022287:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559843840/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202807.022296:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202807.022299:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202807.022303:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559843840:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202807.022327:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559843968:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202807.022343:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202807.022351:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202807/real 1788202807] req@ffff93c0e62b8000 x1875064559843968/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202807.022359:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559843968/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202807.022365:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202807.022366:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202807.022368:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559843968:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202807.022372:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559844096:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202807.022384:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202807.022390:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202807/real 1788202807] req@ffff93c0e62ba680 x1875064559844096/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202807.022397:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559844096/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202807.022404:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202807.022406:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202807.022408:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559844096:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202807.726008:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202808.750080:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202809.773960:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202810.798023:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202811.629989:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202811.630004:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202811.630106:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202811.630139:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202811.630143:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202811.630178:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202811.821979:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202812.141939:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202812.141955:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202812.141960:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202812.141971:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1943 00000100:00080000:2.0:1788202812.141983:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202812.141992:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202812.142042:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559844224/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202812.142072:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202812.142077:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202812.142087:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202812.142092:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1943 00000100:00080000:2.0:1788202812.142097:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202812.142102:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202812.142113:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202812.142117:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559844224:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202812.142131:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559844352/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202812.142145:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202812.142149:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202812.142151:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202812.142156:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1943 00000100:00080000:2.0:1788202812.142161:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202812.142165:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202812.142171:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000800:00000100:0.0:1788202812.142182:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788202812.142187:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559844480/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202812.142220:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202812/real 1788202812] req@ffff93c0e62bb480 x1875064559844224/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202812.142235:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559844224/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202812.142244:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202812.142247:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202812.142250:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559844224:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202812.142274:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559844352:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202812.142289:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202812.142298:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202812/real 1788202812] req@ffff93c0e62baa00 x1875064559844352/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202812.142307:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559844352/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202812.142313:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202812.142316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202812.142318:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559844352:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202812.142322:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559844480:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202812.142333:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202812.142345:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202812/real 1788202812] req@ffff93c0e62ba680 x1875064559844480/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202812.142354:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559844480/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202812.142362:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202812.142364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202812.142366:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559844480:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202812.845983:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202813.869992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202814.894032:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202815.917929:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202816.750069:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202816.750085:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202816.750199:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202816.750229:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202816.750235:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202816.750269:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202816.942026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202817.261958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202817.261966:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202817.261970:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202817.261979:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1948 00000100:00080000:0.0:1788202817.261988:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202817.261994:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202817.262028:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559844608/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202817.262064:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202817.262066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202817.262068:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202817.262071:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1948 00000100:00080000:0.0:1788202817.262074:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202817.262076:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202817.262086:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202817.262100:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559844736/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202817.262111:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202817.262116:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202817.262119:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202817.262124:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1948 00000100:00100000:3.0:1788202817.262129:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559844608:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202817.262130:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202817.262139:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202817.262144:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202817.262155:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559844864/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202817.262198:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202817.262237:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202817/real 1788202817] req@ffff93c0e9ce4380 x1875064559844608/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202817.262256:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559844608/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202817.262267:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202817.262282:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202817.262290:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559844608:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202817.262299:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559844736:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202817.262323:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202817.262335:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202817/real 1788202817] req@ffff93c0e9ce4700 x1875064559844736/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202817.262349:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559844736/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202817.262358:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202817.262362:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202817.262366:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559844736:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202817.262390:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559844864:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202817.262409:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202817.262420:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202817/real 1788202817] req@ffff93c0e9ce7480 x1875064559844864/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202817.262433:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559844864/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202817.262444:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202817.262447:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202817.262451:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559844864:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202817.965987:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202818.989998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202820.014027:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202821.038026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202821.870035:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202821.870048:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202821.870136:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202821.870165:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202821.870169:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202821.870190:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202822.061989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202822.381947:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202822.381958:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202822.381963:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202822.381974:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1953 00000100:00080000:3.0:1788202822.381984:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202822.381990:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202822.382022:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ca80 x1875064559844992/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202822.382045:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202822.382048:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202822.382049:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202822.382052:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1953 00000100:00080000:3.0:1788202822.382054:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202822.382057:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202822.382065:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202822.382073:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d500 x1875064559845120/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202822.382081:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202822.382083:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202822.382085:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202822.382090:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1953 00000100:00080000:3.0:1788202822.382093:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788202822.382094:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559844992:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202822.382096:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202822.382102:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202822.382111:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559845248/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202822.382166:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202822.382221:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202822/real 1788202822] req@ffff93c1c4a4ca80 x1875064559844992/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202822.382244:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ca80 x1875064559844992/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202822.382256:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202822.382261:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202822.382265:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559844992:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202822.382288:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559845120:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202822.382311:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202822.382324:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202822/real 1788202822] req@ffff93c1c4a4d500 x1875064559845120/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202822.382337:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d500 x1875064559845120/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202822.382346:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202822.382349:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202822.382353:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559845120:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202822.382360:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559845248:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202822.382380:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202822.382391:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202822/real 1788202822] req@ffff93c1c4a4d880 x1875064559845248/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202822.382411:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559845248/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202822.382422:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202822.382425:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202822.382429:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559845248:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202823.085961:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202824.109989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202825.134053:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202826.157992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202826.990037:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202826.990052:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202826.990148:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202826.990175:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202826.990178:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202826.990206:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202827.181981:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202827.502020:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202827.502032:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202827.502036:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202827.502047:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1958 00000100:00080000:2.0:1788202827.502059:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202827.502066:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202827.502107:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559845376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202827.502144:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202827.502148:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202827.502150:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202827.502155:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1958 00000100:00080000:2.0:1788202827.502159:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202827.502163:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202827.502173:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202827.502189:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559845504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202827.502203:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202827.502207:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202827.502209:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788202827.502213:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559845376:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202827.502217:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1958 00000100:00080000:2.0:1788202827.502225:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202827.502231:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202827.502237:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202827.502252:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559845632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202827.502304:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202827.502347:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202827/real 1788202827] req@ffff93c0e62b8000 x1875064559845376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202827.502365:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559845376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202827.502376:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202827.502380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202827.502385:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559845376:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202827.502416:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559845504:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202827.502457:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202827.502471:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202827/real 1788202827] req@ffff93c0e62b8380 x1875064559845504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202827.502485:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559845504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202827.502494:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202827.502498:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202827.502502:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559845504:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202827.502508:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559845632:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202827.502527:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202827.502537:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202827/real 1788202827] req@ffff93c0e62b8700 x1875064559845632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202827.502563:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559845632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202827.502575:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202827.502579:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202827.502582:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559845632:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202828.206035:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202829.229980:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202830.253982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202831.277966:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202832.110117:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202832.110141:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202832.110299:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202832.110338:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202832.110344:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202832.110381:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202832.301971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202832.621963:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202832.621972:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202832.621975:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202832.621984:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1963 00000100:00080000:3.0:1788202832.621995:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202832.622003:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202832.622044:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559845760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202832.622085:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202832.622089:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202832.622091:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202832.622096:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1963 00000100:00080000:3.0:1788202832.622100:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202832.622104:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202832.622113:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202832.622128:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559845888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202832.622143:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559845760:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788202832.622151:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202832.622156:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202832.622158:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202832.622163:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1963 00000100:00080000:3.0:1788202832.622166:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202832.622171:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202832.622176:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202832.622190:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559846016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202832.622210:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202832.622256:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202832/real 1788202832] req@ffff93c1c4a4f800 x1875064559845760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202832.622280:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559845760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202832.622291:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202832.622295:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202832.622298:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559845760:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202832.622316:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559845888:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202832.622332:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202832.622340:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202832/real 1788202832] req@ffff93c1c4a4d180 x1875064559845888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202832.622348:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559845888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202832.622354:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202832.622356:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202832.622358:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559845888:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202832.622362:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559846016:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202832.622373:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202832.622378:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202832/real 1788202832] req@ffff93c1c4a4e300 x1875064559846016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202832.622386:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559846016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202832.622397:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202832.622400:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202832.622402:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559846016:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202833.325953:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202834.350036:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202835.373984:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202836.397993:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202837.230012:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202837.230026:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202837.230133:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202837.230165:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202837.230170:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202837.230202:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202837.421990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202837.741989:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202837.742005:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202837.742011:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202837.742025:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1968 00000100:00080000:1.0:1788202837.742041:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202837.742050:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202837.742108:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559846144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202837.742138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202837.742141:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202837.742143:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202837.742146:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1968 00000100:00080000:1.0:1788202837.742149:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202837.742152:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202837.742164:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202837.742176:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559846272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202837.742184:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202837.742187:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202837.742190:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788202837.742192:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559846144:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202837.742195:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1968 00000100:00080000:1.0:1788202837.742200:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202837.742204:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202837.742207:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202837.742215:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559846400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202837.742331:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559846272:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202837.742369:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559846400:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202838.446050:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202839.470026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202840.494040:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788202840.814138:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788202840.814179:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788202840.814192:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788202840.814225:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788202840.814235:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788202840.814286:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202837/real 1788202840] req@ffff93c1c5034e00 x1875064559846272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202840.814309:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559846272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202840.814331:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202840.814336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202840.814340:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559846272:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202840.814348:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202837/real 1788202840] req@ffff93c1c5037100 x1875064559846400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202840.814358:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559846400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202840.814366:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202840.814369:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202840.814371:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559846400:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788202840.814375:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202837/real 1788202840] req@ffff93c1c5034a80 x1875064559846144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202840.814382:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559846144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202840.814389:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202840.814392:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202840.814396:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559846144:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788202841.518023:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202842.350059:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202842.350078:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202842.350194:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202842.350219:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202842.350222:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202842.350239:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202842.541995:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202842.862095:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202842.862108:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202842.862113:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202842.862127:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1973 00000100:00080000:1.0:1788202842.862142:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202842.862151:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202842.862204:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559846528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202842.862243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202842.862248:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202842.862251:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202842.862256:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1973 00000100:00080000:1.0:1788202842.862260:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202842.862265:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202842.862281:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202842.862319:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559846656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202842.862332:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788202842.862337:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559846528:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202842.862338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202842.862342:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202842.862354:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1973 00000100:00080000:1.0:1788202842.862358:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202842.862363:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202842.862370:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202842.862399:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559846784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202842.862445:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202842.862523:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202842/real 1788202842] req@ffff93c1c5036680 x1875064559846528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202842.862546:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559846528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202842.862560:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202842.862565:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202842.862571:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559846528:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202842.862604:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559846656:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202842.862633:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202842.862649:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202842/real 1788202842] req@ffff93c1c5035c00 x1875064559846656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202842.862663:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559846656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202842.862672:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202842.862676:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202842.862679:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559846656:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202842.862686:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559846784:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202842.862708:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202842.862721:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202842/real 1788202842] req@ffff93c1c5037800 x1875064559846784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202842.862733:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559846784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202842.862744:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202842.862747:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202842.862752:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559846784:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202843.566040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202844.589945:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202845.614026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202846.638002:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202847.470099:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202847.470119:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202847.470251:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202847.470274:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202847.470277:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202847.470297:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202847.662003:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202847.981945:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202847.981958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202847.981963:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202847.981974:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1978 00000100:00080000:0.0:1788202847.981987:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202847.981996:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202847.982047:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559846912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202847.982085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202847.982089:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202847.982091:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202847.982096:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1978 00000100:00080000:0.0:1788202847.982101:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202847.982106:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202847.982118:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202847.982134:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559847040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202847.982145:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559846912:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202847.982150:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202847.982156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202847.982158:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202847.982162:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1978 00000100:00080000:0.0:1788202847.982165:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202847.982168:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202847.982172:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202847.982183:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559847168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202847.982258:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202847.982324:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202847/real 1788202847] req@ffff93c0e9ce7480 x1875064559846912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202847.982347:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559846912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202847.982360:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202847.982366:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202847.982371:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559846912:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202847.982400:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559847040:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202847.982437:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202847.982453:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202847/real 1788202847] req@ffff93c0e9ce4700 x1875064559847040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202847.982466:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559847040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202847.982475:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202847.982478:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202847.982482:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559847040:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202847.982489:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559847168:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202847.982510:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202847.982521:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202847/real 1788202847] req@ffff93c0e9ce4380 x1875064559847168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202847.982534:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559847168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202847.982553:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202847.982557:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202847.982561:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559847168:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202848.685950:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202849.710049:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202850.733995:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202851.758052:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202852.590024:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202852.590043:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202852.590206:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202852.590231:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202852.590235:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202852.590254:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202852.782023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202853.101969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202853.101980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202853.101984:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202853.101992:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1983 00000100:00080000:0.0:1788202853.102002:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202853.102009:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202853.102049:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559847296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202853.102072:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202853.102076:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202853.102077:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202853.102081:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1983 00000100:00080000:0.0:1788202853.102085:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202853.102090:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202853.102100:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202853.102142:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559847296:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202853.102145:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559847424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202853.102156:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202853.102159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202853.102160:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202853.102164:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1983 00000100:00080000:0.0:1788202853.102168:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202853.102171:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202853.102176:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202853.102185:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559847552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202853.102202:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202853.102237:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202853/real 1788202853] req@ffff93c0e9ce6a00 x1875064559847296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202853.102250:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559847296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202853.102257:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202853.102260:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202853.102263:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559847296:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202853.102282:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559847424:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202853.102296:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202853.102305:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202853/real 1788202853] req@ffff93c0e9ce4000 x1875064559847424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202853.102313:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559847424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202853.102318:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202853.102320:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202853.102323:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559847424:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202853.102326:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559847552:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202853.102336:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202853.102342:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202853/real 1788202853] req@ffff93c0e9ce6680 x1875064559847552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202853.102349:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559847552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202853.102360:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202853.102362:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202853.102364:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559847552:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202853.805959:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202854.829980:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202855.853970:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202856.877942:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202857.710006:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202857.710024:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202857.710147:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202857.710172:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202857.710176:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202857.710200:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202857.901954:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202858.221958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202858.221967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202858.221971:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202858.221980:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1989 00000100:00080000:0.0:1788202858.221991:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202858.221998:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202858.222076:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559847680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202858.222109:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202858.222114:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202858.222116:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202858.222120:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1989 00000100:00080000:0.0:1788202858.222125:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202858.222130:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202858.222145:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202858.222158:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559847808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202858.222167:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559847680:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202858.222180:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202858.222183:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202858.222185:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202858.222189:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1989 00000100:00080000:0.0:1788202858.222191:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202858.222194:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202858.222198:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202858.222208:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559847936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202858.222236:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202858.222279:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202858/real 1788202858] req@ffff93c0e9ce7b80 x1875064559847680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202858.222292:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559847680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202858.222301:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202858.222311:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202858.222314:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559847680:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202858.222333:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559847808:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202858.222349:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202858.222357:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202858/real 1788202858] req@ffff93c0e9ce5c00 x1875064559847808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202858.222365:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559847808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202858.222371:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202858.222373:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202858.222375:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559847808:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202858.222380:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559847936:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202858.222391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202858.222397:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202858/real 1788202858] req@ffff93c0e9ce6d80 x1875064559847936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202858.222405:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559847936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202858.222411:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202858.222413:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202858.222415:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559847936:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202858.925962:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202859.949935:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202860.974008:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202861.997936:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202862.830053:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202862.830078:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202862.830196:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202862.830227:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202862.830231:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202862.830262:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202863.021897:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202863.341976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202863.341986:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202863.341990:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202863.342000:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1994 00000100:00080000:2.0:1788202863.342010:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202863.342018:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202863.342053:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559848064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202863.342081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202863.342085:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202863.342086:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202863.342090:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1994 00000100:00080000:2.0:1788202863.342093:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202863.342097:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202863.342106:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202863.342117:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559848192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202863.342127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202863.342131:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202863.342140:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788202863.342143:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559848064:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202863.342147:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1994 00000100:00080000:2.0:1788202863.342153:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202863.342156:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202863.342161:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202863.342173:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559848320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202863.342208:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202863.342239:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202863/real 1788202863] req@ffff93c0e62bb100 x1875064559848064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202863.342250:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559848064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202863.342257:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202863.342259:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202863.342261:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559848064:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202863.342266:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559848192:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202863.342279:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202863.342286:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202863/real 1788202863] req@ffff93c0e62bbb80 x1875064559848192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202863.342293:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559848192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202863.342298:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202863.342300:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202863.342302:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559848192:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202863.342320:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559848320:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202863.342329:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202863.342334:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202863/real 1788202863] req@ffff93c0e62b9c00 x1875064559848320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202863.342341:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559848320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202863.342347:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202863.342348:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202863.342350:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559848320:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202864.045974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202865.070013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202866.093977:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202867.117981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202867.949983:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202867.949997:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202867.950104:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202867.950124:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202867.950128:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202867.950148:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202868.141980:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202868.461965:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202868.461973:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202868.461976:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202868.461984:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 1999 00000100:00080000:0.0:1788202868.461992:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202868.461997:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202868.462025:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559848448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202868.462044:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202868.462046:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202868.462047:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202868.462050:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1999 00000100:00080000:0.0:1788202868.462052:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202868.462054:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202868.462061:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202868.462068:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559848576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202868.462075:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202868.462077:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202868.462078:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202868.462080:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 1999 00000100:00080000:0.0:1788202868.462082:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202868.462084:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202868.462087:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202868.462091:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559848448:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202868.462095:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559848704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202868.462144:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202868.462188:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202868/real 1788202868] req@ffff93c0e9ce7100 x1875064559848448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202868.462207:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559848448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202868.462219:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202868.462222:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202868.462224:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559848448:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202868.462229:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559848576:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202868.462246:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202868.462254:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202868/real 1788202868] req@ffff93c0e9ce5f80 x1875064559848576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202868.462261:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559848576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202868.462266:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202868.462268:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202868.462270:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559848576:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202868.462293:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559848704:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202868.462313:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202868.462323:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202868/real 1788202868] req@ffff93c0e9ce4a80 x1875064559848704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202868.462330:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559848704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202868.462337:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202868.462339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202868.462341:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559848704:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202869.165946:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202870.189972:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202871.213962:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202872.237925:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202873.070011:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202873.070023:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202873.070122:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202873.070144:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202873.070147:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202873.070165:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202873.261968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202873.581968:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202873.581978:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202873.581982:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202873.581991:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2004 00000100:00080000:1.0:1788202873.582000:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202873.582007:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202873.582044:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559848832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202873.582067:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202873.582070:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202873.582072:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202873.582076:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2004 00000100:00080000:1.0:1788202873.582079:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202873.582082:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202873.582091:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202873.582102:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559848960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202873.582111:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202873.582114:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202873.582116:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202873.582120:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2004 00000100:00080000:1.0:1788202873.582125:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202873.582126:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559848832:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202873.582132:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202873.582137:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202873.582146:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559849088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202873.582194:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202873.582240:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202873/real 1788202873] req@ffff93c1c5037800 x1875064559848832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202873.582257:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559848832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202873.582268:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202873.582278:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202873.582285:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559848832:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202873.582292:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559848960:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202873.582312:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202873.582322:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202873/real 1788202873] req@ffff93c1c5035c00 x1875064559848960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202873.582333:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559848960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202873.582346:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202873.582350:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202873.582353:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559848960:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202873.582374:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559849088:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202873.582391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202873.582402:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202873/real 1788202873] req@ffff93c1c5036680 x1875064559849088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202873.582413:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559849088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202873.582423:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202873.582426:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202873.582429:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559849088:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202874.285985:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202875.309966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202876.333962:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202877.357962:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202878.189965:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202878.189978:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202878.190077:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202878.190095:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202878.190099:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202878.190116:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202878.381975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202878.701922:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202878.701930:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202878.701933:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202878.701940:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2009 00000100:00080000:0.0:1788202878.701949:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202878.701956:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202878.701997:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559849216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202878.702027:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202878.702031:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202878.702034:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202878.702039:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2009 00000100:00080000:0.0:1788202878.702043:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202878.702046:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202878.702054:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202878.702065:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559849344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202878.702072:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559849216:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788202878.702078:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202878.702082:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202878.702085:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202878.702090:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2009 00000100:00080000:0.0:1788202878.702094:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202878.702105:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202878.702111:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202878.702123:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559849472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202878.702126:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202878.702169:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202878/real 1788202878] req@ffff93c0e9ce7800 x1875064559849216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202878.702188:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559849216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202878.702198:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202878.702202:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202878.702206:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559849216:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202878.702224:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559849344:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202878.702240:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202878.702248:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202878/real 1788202878] req@ffff93c0e9ce5180 x1875064559849344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202878.702255:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559849344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202878.702265:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202878.702266:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202878.702268:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559849344:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202878.702272:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559849472:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202878.702280:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202878.702285:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202878/real 1788202878] req@ffff93c0e9ce5500 x1875064559849472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202878.702292:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559849472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202878.702298:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202878.702300:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202878.702301:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559849472:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202879.405932:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202880.429948:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202881.453939:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202882.477970:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202883.310005:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202883.310017:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202883.310100:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202883.310118:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202883.310121:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202883.310137:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202883.501948:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202883.821923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202883.821930:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202883.821934:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202883.821941:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2014 00000100:00080000:0.0:1788202883.821949:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202883.821955:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202883.821984:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559849600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202883.822005:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202883.822008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202883.822009:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202883.822012:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2014 00000100:00080000:0.0:1788202883.822015:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202883.822017:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202883.822024:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202883.822031:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559849728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202883.822038:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202883.822040:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202883.822041:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202883.822044:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2014 00000100:00080000:0.0:1788202883.822045:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202883.822049:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202883.822052:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202883.822057:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559849856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202883.822102:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559849600:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788202883.822195:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202883.822263:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202883/real 1788202883] req@ffff93c0e9ce4e00 x1875064559849600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202883.822294:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559849600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202883.822317:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202883.822322:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202883.822327:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559849600:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202883.822337:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559849728:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202883.822362:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202883.822376:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202883/real 1788202883] req@ffff93c0e9ce5880 x1875064559849728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202883.822389:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559849728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202883.822398:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202883.822402:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202883.822406:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559849728:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202883.822413:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559849856:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202883.822436:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202883.822449:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202883/real 1788202883] req@ffff93c0e9ce6300 x1875064559849856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202883.822463:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559849856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202883.822475:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202883.822480:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202883.822484:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559849856:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202884.525994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202885.550043:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202886.573981:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202887.597921:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202888.429982:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202888.429994:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202888.430085:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202888.430110:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202888.430114:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202888.430142:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202888.621955:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202888.941958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202888.941966:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202888.941970:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202888.941980:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2019 00000100:00080000:2.0:1788202888.941990:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202888.941997:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202888.942035:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559849984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202888.942057:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202888.942061:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202888.942063:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202888.942067:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2019 00000100:00080000:2.0:1788202888.942071:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202888.942075:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202888.942084:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202888.942096:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559850112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202888.942098:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559849984:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202888.942109:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202888.942112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202888.942113:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202888.942116:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2019 00000100:00080000:2.0:1788202888.942118:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202888.942121:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202888.942125:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202888.942132:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559850240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202888.942165:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202888.942211:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202888/real 1788202888] req@ffff93c0e62b9500 x1875064559849984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202888.942230:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559849984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202888.942241:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202888.942245:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202888.942249:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559849984:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202888.942271:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559850112:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202888.942292:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202888.942303:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202888/real 1788202888] req@ffff93c0e62bad80 x1875064559850112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202888.942315:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559850112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202888.942323:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202888.942326:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202888.942329:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559850112:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202888.942335:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559850240:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202888.942352:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202888.942363:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202888/real 1788202888] req@ffff93c0e62b8a80 x1875064559850240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202888.942374:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559850240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202888.942389:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202888.942393:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202888.942396:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559850240:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202889.645923:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202890.669960:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202891.693940:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202892.717959:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202893.549955:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202893.549966:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202893.550077:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202893.550094:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202893.550097:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202893.550113:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202893.741984:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202894.061968:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202894.061983:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202894.061989:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202894.062002:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2024 00000100:00080000:1.0:1788202894.062016:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202894.062026:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202894.062090:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559850368/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202894.062118:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202894.062121:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202894.062122:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202894.062126:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2024 00000100:00080000:1.0:1788202894.062129:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202894.062132:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202894.062140:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202894.062152:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559850496/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202894.062160:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202894.062164:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202894.062165:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202894.062169:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2024 00000100:00080000:1.0:1788202894.062172:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202894.062179:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202894.062182:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202894.062182:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559850368:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788202894.062192:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559850624/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202894.062264:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202894.062328:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202894/real 1788202894] req@ffff93c1c5034a80 x1875064559850368/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202894.062353:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559850368/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202894.062367:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202894.062372:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202894.062377:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559850368:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202894.062385:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559850496:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202894.062426:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202894.062442:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202894/real 1788202894] req@ffff93c1c5037100 x1875064559850496/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202894.062456:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559850496/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202894.062464:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202894.062468:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202894.062471:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559850496:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202894.062499:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559850624:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202894.062520:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202894.062532:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202894/real 1788202894] req@ffff93c1c5034e00 x1875064559850624/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202894.062545:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559850624/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202894.062555:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202894.062559:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202894.062563:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559850624:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202894.766050:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202895.789998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202896.814016:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202897.837951:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202898.670083:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202898.670105:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202898.670233:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202898.670283:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202898.670289:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202898.670348:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202898.861964:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202899.181963:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202899.181977:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202899.181982:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202899.181994:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2030 00000100:00080000:1.0:1788202899.182007:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202899.182015:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202899.182061:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559850752/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202899.182089:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202899.182092:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202899.182094:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202899.182097:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2030 00000100:00080000:1.0:1788202899.182099:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202899.182102:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202899.182112:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202899.182122:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559850880/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202899.182131:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788202899.182136:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559850752:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202899.182138:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202899.182143:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202899.182150:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2030 00000100:00080000:1.0:1788202899.182154:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202899.182159:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202899.182165:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202899.182182:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559851008/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202899.182325:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202899.182377:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202899/real 1788202899] req@ffff93c1c5034e00 x1875064559850752/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202899.182393:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559850752/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202899.182401:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202899.182404:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202899.182406:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559850752:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202899.182429:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559850880:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202899.182473:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202899.182484:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202899/real 1788202899] req@ffff93c1c5037100 x1875064559850880/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202899.182494:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559850880/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202899.182499:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202899.182501:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202899.182503:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559850880:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202899.182507:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559851008:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202899.182522:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202899.182528:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202899/real 1788202899] req@ffff93c1c5034a80 x1875064559851008/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202899.182534:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559851008/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202899.182541:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202899.182544:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202899.182547:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559851008:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202899.886067:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202900.909980:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202901.933993:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202902.958005:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202903.790073:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202903.790091:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202903.790246:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202903.790273:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202903.790276:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202903.790299:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202903.981983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202904.301980:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202904.301994:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202904.302001:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202904.302015:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2035 00000100:00080000:1.0:1788202904.302028:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202904.302045:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202904.302100:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559851136/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202904.302135:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202904.302139:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202904.302141:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202904.302146:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2035 00000100:00080000:1.0:1788202904.302150:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202904.302155:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202904.302168:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202904.302181:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559851264/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202904.302191:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202904.302194:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202904.302198:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788202904.302200:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559851136:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788202904.302203:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2035 00000100:00080000:1.0:1788202904.302209:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202904.302214:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202904.302236:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202904.302252:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559851392/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202904.302365:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559851264:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202904.302419:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559851392:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202905.005941:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202906.029992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202907.054095:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788202907.374258:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788202907.376139:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788202907.376159:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202907.376205:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202907.376215:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788202907.376763:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202904/real 1788202907] req@ffff93c1c5035c00 x1875064559851264/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202907.376793:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559851264/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202907.376804:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202907.376808:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202907.376812:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559851264:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202907.376820:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202904/real 1788202907] req@ffff93c1c5037800 x1875064559851392/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202907.376863:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559851392/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202907.376874:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202907.376878:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202907.376882:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559851392:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788202907.376892:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202904/real 1788202907] req@ffff93c1c5036680 x1875064559851136/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202907.376903:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559851136/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202907.376909:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202907.376912:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202907.376915:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559851136:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788202908.077940:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202908.910040:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202908.910057:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202908.910158:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202908.910175:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202908.910179:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202908.910196:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202909.101967:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202909.421994:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202909.422005:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202909.422010:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202909.422022:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2040 00000100:00080000:2.0:1788202909.422035:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202909.422042:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202909.422085:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559851520/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202909.422123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202909.422142:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202909.422145:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202909.422151:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2040 00000100:00080000:2.0:1788202909.422161:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202909.422167:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202909.422180:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202909.422191:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559851520:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202909.422199:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559851648/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202909.422214:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202909.422219:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202909.422221:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202909.422226:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2040 00000100:00080000:2.0:1788202909.422230:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202909.422235:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202909.422242:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202909.422266:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559851776/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202909.422294:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202909.422349:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202909/real 1788202909] req@ffff93c0e62b9f80 x1875064559851520/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202909.422371:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559851520/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202909.422383:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202909.422394:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202909.422399:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559851520:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202909.422435:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559851648:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202909.422462:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202909.422477:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202909/real 1788202909] req@ffff93c0e62bb800 x1875064559851648/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202909.422496:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559851648/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202909.422504:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202909.422507:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202909.422510:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559851648:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202909.422517:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559851776:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202909.422539:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202909.422549:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202909/real 1788202909] req@ffff93c0e62b8a80 x1875064559851776/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202909.422568:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559851776/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202909.422579:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202909.422583:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202909.422586:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559851776:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202910.125958:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202911.149918:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202912.173940:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202913.197930:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202914.030021:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202914.030039:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202914.030182:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202914.030205:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202914.030208:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202914.030239:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202914.222004:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202914.541996:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202914.542010:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202914.542017:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202914.542032:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2045 00000100:00080000:2.0:1788202914.542074:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202914.542085:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202914.542140:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559851904/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202914.542178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202914.542182:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202914.542184:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202914.542196:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2045 00000100:00080000:2.0:1788202914.542200:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202914.542205:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202914.542218:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202914.542234:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559852032/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202914.542246:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202914.542251:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202914.542253:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788202914.542259:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559851904:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202914.542261:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2045 00000100:00080000:2.0:1788202914.542270:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202914.542276:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202914.542281:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202914.542293:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559852160/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202914.542352:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202914.542429:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202914/real 1788202914] req@ffff93c0e62bad80 x1875064559851904/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202914.542576:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559851904/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202914.542590:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202914.542594:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202914.542598:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559851904:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202914.542634:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559852032:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202914.542690:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202914.542708:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202914/real 1788202914] req@ffff93c0e62b9500 x1875064559852032/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202914.542717:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559852032/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202914.542726:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202914.542728:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202914.542730:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559852032:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202914.542735:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559852160:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202914.542745:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202914.542751:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202914/real 1788202914] req@ffff93c0e62b9c00 x1875064559852160/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202914.542757:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559852160/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202914.542764:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202914.542766:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202914.542767:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559852160:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202915.245993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202916.270050:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202917.293959:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202918.317951:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202919.149944:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202919.149957:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202919.150058:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202919.150076:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202919.150079:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202919.150099:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202919.341993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202919.661933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202919.661943:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202919.661946:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202919.661977:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2050 00000100:00080000:2.0:1788202919.661988:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202919.661994:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202919.662027:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559852288/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202919.662051:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202919.662055:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202919.662056:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202919.662060:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2050 00000100:00080000:2.0:1788202919.662063:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202919.662066:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202919.662074:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202919.662085:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559852416/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202919.662099:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559852288:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202919.662102:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202919.662108:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202919.662109:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202919.662113:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2050 00000100:00080000:2.0:1788202919.662115:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202919.662117:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202919.662121:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202919.662129:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559852544/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202919.662162:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202919.662208:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202919/real 1788202919] req@ffff93c0e62bbb80 x1875064559852288/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202919.662226:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559852288/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202919.662235:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202919.662240:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202919.662244:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559852288:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202919.662264:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559852416:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202919.662285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202919.662297:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202919/real 1788202919] req@ffff93c0e62bb100 x1875064559852416/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202919.662308:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559852416/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202919.662316:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202919.662319:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202919.662322:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559852416:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202919.662328:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559852544:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202919.662346:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202919.662356:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202919/real 1788202919] req@ffff93c0e62b8700 x1875064559852544/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202919.662367:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559852544/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202919.662376:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202919.662379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202919.662382:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559852544:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202920.365958:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202921.389961:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202922.413942:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202923.437976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788202924.269990:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202924.270001:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202924.270118:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788202924.270144:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788202924.270148:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788202924.270168:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202924.461968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788202924.781960:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202924.781970:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788202924.781974:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202924.781984:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2055 00000100:00080000:0.0:1788202924.781993:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202924.782001:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788202924.782035:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559852672/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202924.782061:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202924.782071:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788202924.782073:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202924.782077:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2055 00000100:00080000:0.0:1788202924.782081:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202924.782086:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202924.782096:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202924.782101:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559852672:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202924.782116:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559852800/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788202924.782129:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788202924.782133:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788202924.782135:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788202924.782139:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2055 00000100:00080000:0.0:1788202924.782142:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788202924.782145:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788202924.782150:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202924.782161:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559852928/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202924.782206:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202924.782254:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202924/real 1788202924] req@ffff93c0e9ce6300 x1875064559852672/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202924.782280:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559852672/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202924.782291:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202924.782294:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202924.782298:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559852672:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202924.782321:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559852800:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202924.782345:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202924.782358:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202924/real 1788202924] req@ffff93c0e9ce5880 x1875064559852800/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202924.782370:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559852800/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202924.782378:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202924.782381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202924.782384:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559852800:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202924.782391:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559852928:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202924.782410:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202924.782422:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202924/real 1788202924] req@ffff93c0e9ce4e00 x1875064559852928/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202924.782434:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559852928/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202924.782450:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202924.782454:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202924.782457:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559852928:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202925.485956:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202926.509936:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202927.533982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202928.557936:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202929.390072:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202929.390086:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202929.390180:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202929.390209:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202929.390214:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202929.390239:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202929.581959:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202929.901964:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202929.901972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202929.901975:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202929.901983:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2060 00000100:00080000:2.0:1788202929.901992:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202929.901997:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202929.902028:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559853056/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202929.902049:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202929.902051:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202929.902053:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202929.902055:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2060 00000100:00080000:2.0:1788202929.902058:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202929.902062:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202929.902072:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202929.902085:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559853184/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202929.902094:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559853056:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202929.902119:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202929.902122:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202929.902124:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202929.902131:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2060 00000100:00080000:2.0:1788202929.902135:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202929.902140:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202929.902146:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202929.902162:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559853312/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202929.902178:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202929.902237:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202929/real 1788202929] req@ffff93c0e62b8380 x1875064559853056/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202929.902260:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559853056/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202929.902269:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202929.902272:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202929.902275:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559853056:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202929.902301:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559853184:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202929.902320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202929.902329:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202929/real 1788202929] req@ffff93c0e62b8000 x1875064559853184/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202929.902336:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559853184/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202929.902341:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202929.902344:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202929.902345:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559853184:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202929.902351:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559853312:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202929.902362:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202929.902368:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202929/real 1788202929] req@ffff93c0e62ba680 x1875064559853312/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202929.902375:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559853312/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202929.902381:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202929.902383:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202929.902384:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559853312:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202930.605934:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202931.629956:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202932.653950:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202933.677985:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202934.509985:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202934.510000:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202934.510085:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202934.510104:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202934.510107:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202934.510125:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202934.701931:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202935.021906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202935.021915:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202935.021919:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202935.021928:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2065 00000100:00080000:2.0:1788202935.021939:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202935.021946:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202935.021992:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559853440/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202935.022016:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202935.022019:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202935.022021:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202935.022025:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2065 00000100:00080000:2.0:1788202935.022028:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202935.022032:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202935.022041:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202935.022054:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559853568/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202935.022058:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559853440:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202935.022072:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202935.022076:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202935.022078:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202935.022083:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2065 00000100:00080000:2.0:1788202935.022087:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202935.022091:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202935.022096:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202935.022108:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559853696/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202935.022115:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202935.022146:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202935/real 1788202935] req@ffff93c0e62baa00 x1875064559853440/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202935.022157:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559853440/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202935.022164:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202935.022166:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202935.022169:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559853440:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202935.022184:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559853568:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202935.022197:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202935.022204:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202935/real 1788202935] req@ffff93c0e62bb480 x1875064559853568/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202935.022211:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559853568/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202935.022217:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202935.022220:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202935.022223:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559853568:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202935.022228:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559853696:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202935.022246:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202935.022256:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202935/real 1788202935] req@ffff93c0e62ba680 x1875064559853696/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202935.022266:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559853696/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202935.022275:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202935.022278:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202935.022281:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559853696:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202935.725949:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202936.749991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202937.773933:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202938.798019:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202939.629983:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202939.629998:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202939.630106:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202939.630135:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202939.630140:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202939.630168:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202939.821972:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202940.141972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202940.141981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202940.141984:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202940.141992:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2071 00000100:00080000:2.0:1788202940.142001:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202940.142009:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202940.142044:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559853824/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202940.142073:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202940.142078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202940.142080:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202940.142085:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2071 00000100:00080000:2.0:1788202940.142089:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202940.142094:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202940.142105:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788202940.142120:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559853824:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202940.142122:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559853952/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202940.142138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202940.142142:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202940.142145:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202940.142150:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2071 00000100:00080000:2.0:1788202940.142169:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202940.142174:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000800:00000100:3.0:1788202940.142177:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000008:00100000:2.0:1788202940.142184:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202940.142201:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559854080/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202940.142214:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202940/real 1788202940] req@ffff93c0e62b8000 x1875064559853824/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202940.142228:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559853824/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202940.142241:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202940.142244:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202940.142247:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559853824:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202940.142266:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559853952:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202940.142281:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202940.142289:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202940/real 1788202940] req@ffff93c0e62b8380 x1875064559853952/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202940.142297:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559853952/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202940.142302:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202940.142304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202940.142307:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559853952:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202940.142310:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559854080:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202940.142321:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202940.142326:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202940/real 1788202940] req@ffff93c0e62b8700 x1875064559854080/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202940.142333:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559854080/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202940.142340:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202940.142342:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202940.142344:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559854080:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202940.845926:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202941.870026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202942.894005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202943.918011:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202944.749995:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202944.750008:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202944.750088:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202944.750106:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202944.750109:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202944.750126:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202944.941982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202945.261982:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202945.261993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202945.261998:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202945.262009:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2076 00000100:00080000:2.0:1788202945.262020:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202945.262028:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202945.262068:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559854208/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202945.262101:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202945.262105:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202945.262107:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202945.262112:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2076 00000100:00080000:2.0:1788202945.262116:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202945.262121:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202945.262133:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202945.262149:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559854336/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202945.262163:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202945.262167:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202945.262169:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202945.262174:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2076 00000100:00080000:2.0:1788202945.262180:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788202945.262184:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559854208:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202945.262189:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202945.262203:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202945.262224:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559854464/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202945.262290:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202945.262343:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202945/real 1788202945] req@ffff93c0e62bb100 x1875064559854208/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202945.262360:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559854208/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202945.262370:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202945.262374:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202945.262377:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559854208:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202945.262383:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559854336:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202945.262403:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202945.262411:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202945/real 1788202945] req@ffff93c0e62bbb80 x1875064559854336/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202945.262420:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559854336/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202945.262426:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202945.262428:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202945.262430:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559854336:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202945.262456:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559854464:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202945.262468:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202945.262474:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202945/real 1788202945] req@ffff93c0e62b9c00 x1875064559854464/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202945.262483:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559854464/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202945.262490:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202945.262492:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202945.262494:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559854464:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202945.965993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202946.989977:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202948.014014:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202949.037984:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202949.870014:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202949.870033:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202949.870160:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202949.870182:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202949.870185:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202949.870204:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202950.061958:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202950.381966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202950.381982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202950.381988:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202950.382001:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2081 00000100:00080000:2.0:1788202950.382017:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202950.382026:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202950.382082:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559854592/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202950.382109:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202950.382112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202950.382113:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202950.382118:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2081 00000100:00080000:2.0:1788202950.382122:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202950.382127:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202950.382140:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202950.382157:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559854720/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202950.382171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202950.382178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202950.382181:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788202950.382182:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559854592:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202950.382190:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2081 00000100:00080000:2.0:1788202950.382197:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202950.382200:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202950.382205:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202950.382218:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559854848/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202950.382255:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202950.382307:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202950/real 1788202950] req@ffff93c0e62b9500 x1875064559854592/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202950.382324:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559854592/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202950.382333:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202950.382337:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202950.382340:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559854592:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202950.382362:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559854720:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202950.382379:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202950.382387:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202950/real 1788202950] req@ffff93c0e62bad80 x1875064559854720/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202950.382396:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559854720/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202950.382407:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202950.382409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202950.382411:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559854720:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202950.382441:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559854848:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202950.382459:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202950.382467:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202950/real 1788202950] req@ffff93c0e62b8a80 x1875064559854848/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202950.382475:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559854848/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202950.382483:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202950.382485:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202950.382487:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559854848:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202951.085990:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202952.109973:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202953.133960:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202954.157997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202954.990025:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202954.990039:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202954.990131:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202954.990159:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202954.990163:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202954.990185:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202955.181972:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202955.501950:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202955.501961:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202955.501965:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202955.501977:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2086 00000100:00080000:2.0:1788202955.501987:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202955.501994:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202955.502029:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559854976/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202955.502055:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202955.502058:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202955.502059:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202955.502062:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2086 00000100:00080000:2.0:1788202955.502064:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202955.502067:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202955.502075:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202955.502084:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559855104/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202955.502091:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202955.502094:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202955.502095:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202955.502098:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2086 00000100:00080000:2.0:1788202955.502101:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202955.502105:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202955.502111:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202955.502111:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559854976:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202955.502123:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559855232/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202955.502174:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202955.502205:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202955/real 1788202955] req@ffff93c0e62bb800 x1875064559854976/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202955.502217:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559854976/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202955.502225:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202955.502228:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202955.502230:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559854976:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202955.502245:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559855104:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202955.502260:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202955.502269:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202955/real 1788202955] req@ffff93c0e62b9f80 x1875064559855104/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202955.502277:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559855104/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202955.502282:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202955.502285:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202955.502287:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559855104:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202955.502306:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559855232:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202955.502317:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202955.502324:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202955/real 1788202955] req@ffff93c0e62b8a80 x1875064559855232/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202955.502356:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559855232/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202955.502364:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202955.502366:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202955.502368:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559855232:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202956.206010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202957.229989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202958.254019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202959.277955:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202960.110029:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202960.110043:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202960.110130:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202960.110148:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202960.110151:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202960.110178:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202960.302004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202960.621901:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202960.621910:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202960.621914:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202960.621921:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2091 00000100:00080000:2.0:1788202960.621931:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202960.621937:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202960.621971:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559855360/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202960.621993:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202960.622002:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202960.622003:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202960.622006:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2091 00000100:00080000:2.0:1788202960.622009:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202960.622013:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202960.622021:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202960.622030:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559855488/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202960.622038:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559855360:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202960.622042:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202960.622045:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202960.622047:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202960.622051:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2091 00000100:00080000:2.0:1788202960.622053:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202960.622055:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202960.622059:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202960.622067:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559855616/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788202960.622096:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202960.622134:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202960/real 1788202960] req@ffff93c0e62bad80 x1875064559855360/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202960.622155:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559855360/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202960.622164:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202960.622167:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202960.622170:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559855360:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202960.622189:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559855488:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202960.622205:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202960.622214:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202960/real 1788202960] req@ffff93c0e62b9500 x1875064559855488/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202960.622222:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559855488/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202960.622235:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202960.622237:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202960.622250:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559855488:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202960.622255:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559855616:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202960.622268:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202960.622276:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202960/real 1788202960] req@ffff93c0e62b9c00 x1875064559855616/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202960.622284:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559855616/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202960.622295:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202960.622297:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202960.622300:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559855616:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202961.326004:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202962.349975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202963.373972:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202964.397970:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202965.229958:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202965.229974:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202965.230063:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202965.230090:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202965.230095:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202965.230128:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202965.421943:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202965.741931:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202965.741941:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202965.741945:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202965.741953:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2096 00000100:00080000:2.0:1788202965.741962:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202965.741968:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202965.742002:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559855744/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202965.742027:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202965.742030:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202965.742032:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202965.742035:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2096 00000100:00080000:2.0:1788202965.742037:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202965.742040:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202965.742047:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202965.742055:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559855872/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202965.742062:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202965.742064:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202965.742065:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202965.742068:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2096 00000100:00080000:2.0:1788202965.742070:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202965.742072:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202965.742074:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202965.742080:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559856000/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788202965.742115:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559855744:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788202965.742195:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202965.742241:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202965/real 1788202965] req@ffff93c0e62bbb80 x1875064559855744/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202965.742257:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559855744/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202965.742266:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202965.742269:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202965.742272:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559855744:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202965.742278:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559855872:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788202965.742300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202965.742308:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202965/real 1788202965] req@ffff93c0e62bb100 x1875064559855872/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202965.742316:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559855872/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202965.742322:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202965.742324:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202965.742326:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559855872:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788202965.742331:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559856000:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788202965.742341:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788202965.742347:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202965/real 1788202965] req@ffff93c0e62b8700 x1875064559856000/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788202965.742354:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559856000/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788202965.742361:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788202965.742363:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788202965.742364:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559856000:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202966.445957:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202967.469922:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202968.493963:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202969.517974:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788202970.350042:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202970.350056:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202970.350156:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788202970.350179:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788202970.350182:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788202970.350200:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202970.541947:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788202970.861953:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202970.861963:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788202970.861967:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202970.861977:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2101 00000100:00080000:3.0:1788202970.861988:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202970.861995:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788202970.862027:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559856128/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202970.862071:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202970.862076:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788202970.862078:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202970.862083:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2101 00000100:00080000:3.0:1788202970.862087:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202970.862092:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202970.862102:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202970.862118:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559856128:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788202970.862119:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559856256/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788202970.862133:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788202970.862136:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788202970.862137:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788202970.862141:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2101 00000100:00080000:3.0:1788202970.862144:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788202970.862147:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788202970.862151:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788202970.862161:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559856384/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202970.862223:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559856256:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202970.862261:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559856384:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202971.566016:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202972.589986:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202973.613973:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788202973.934064:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788202973.934102:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788202973.934116:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202973.934156:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788202973.934167:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788202973.934211:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202970/real 1788202973] req@ffff93c1c4a4c700 x1875064559856256/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202973.934237:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559856256/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788202995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202973.934249:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202973.934254:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202973.934259:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559856256:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202973.934267:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202970/real 1788202973] req@ffff93c1c4a4c380 x1875064559856384/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202973.934298:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559856384/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788202995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202973.934308:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202973.934313:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202973.934317:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559856384:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788202973.934324:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202970/real 1788202973] req@ffff93c1c4a4f480 x1875064559856128/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202973.934337:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559856128/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788202995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202973.934344:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202973.934348:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202973.934352:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559856128:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788202974.637950:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202975.470956:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202975.470971:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202975.471087:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202975.471118:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202975.471123:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202975.471158:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202975.661965:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202975.982004:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202975.982021:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202975.982025:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202975.982040:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2106 00000100:00080000:2.0:1788202975.982051:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202975.982059:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202975.982097:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559856512/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202975.982127:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202975.982131:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202975.982133:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202975.982138:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2106 00000100:00080000:2.0:1788202975.982143:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202975.982148:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202975.982158:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202975.982172:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559856640/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202975.982184:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202975.982191:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788202975.982193:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559856512:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202975.982195:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202975.982207:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2106 00000100:00080000:2.0:1788202975.982213:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202975.982218:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202975.982225:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202975.982238:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559856768/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202975.982306:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202975.982373:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202975/real 1788202975] req@ffff93c0e62b8380 x1875064559856512/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202975.982401:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559856512/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202975.982413:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202975.982418:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202975.982422:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559856512:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202975.982463:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559856640:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202975.982494:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202975.982510:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202975/real 1788202975] req@ffff93c0e62b8000 x1875064559856640/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202975.982524:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559856640/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202975.982534:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202975.982538:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202975.982542:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559856640:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202975.982550:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559856768:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202975.982574:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202975.982588:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202975/real 1788202975] req@ffff93c0e62ba680 x1875064559856768/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202975.982601:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559856768/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202975.982613:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202975.982617:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202975.982620:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559856768:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202976.685990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202977.710043:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202978.733958:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202979.758004:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202980.590029:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202980.590043:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202980.590145:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202980.590166:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202980.590169:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202980.590188:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202980.782020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202981.101973:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202981.101981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202981.101985:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202981.101993:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2111 00000100:00080000:1.0:1788202981.102003:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202981.102011:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202981.102051:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559856896/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202981.102078:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202981.102081:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202981.102083:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202981.102087:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2111 00000100:00080000:1.0:1788202981.102091:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202981.102096:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202981.102106:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788202981.102124:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559856896:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788202981.102132:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559857024/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202981.102144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202981.102146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202981.102148:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202981.102151:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2111 00000100:00080000:1.0:1788202981.102155:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202981.102158:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202981.102164:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202981.102174:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559857152/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202981.103250:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202981.103288:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202981/real 1788202981] req@ffff93c1c5036680 x1875064559856896/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202981.103302:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559856896/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202981.103310:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202981.103313:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202981.103316:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559856896:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202981.103334:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559857024:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202981.103352:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202981.103360:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202981/real 1788202981] req@ffff93c1c5037800 x1875064559857024/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202981.103368:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559857024/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202981.103374:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202981.103376:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202981.103378:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559857024:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202981.103382:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559857152:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202981.103393:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202981.103398:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202981/real 1788202981] req@ffff93c1c5035c00 x1875064559857152/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202981.103406:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559857152/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202981.103417:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202981.103419:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202981.103421:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559857152:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202981.806056:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202982.830021:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202983.854027:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202984.877990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202985.710054:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202985.710066:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202985.710142:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202985.710160:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202985.710163:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202985.710180:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202985.902014:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202986.221937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202986.221946:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202986.221950:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202986.221957:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2117 00000100:00080000:2.0:1788202986.221967:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202986.221973:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202986.222008:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559857280/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202986.222032:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202986.222035:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202986.222037:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202986.222040:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2117 00000100:00080000:2.0:1788202986.222042:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202986.222044:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202986.222053:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202986.222064:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559857408/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202986.222075:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202986.222079:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202986.222081:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202986.222087:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2117 00000100:00080000:2.0:1788202986.222091:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202986.222096:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202986.222102:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788202986.222116:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559857280:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788202986.222123:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559857536/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202986.222176:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202986.222210:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202986/real 1788202986] req@ffff93c0e62bb480 x1875064559857280/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202986.222224:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559857280/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202986.222232:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202986.222235:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202986.222238:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559857280:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202986.222242:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559857408:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202986.222255:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202986.222263:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202986/real 1788202986] req@ffff93c0e62baa00 x1875064559857408/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202986.222270:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559857408/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202986.222276:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202986.222279:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202986.222281:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559857408:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202986.222298:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559857536:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202986.222309:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202986.222315:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202986/real 1788202986] req@ffff93c0e62ba680 x1875064559857536/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202986.222322:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559857536/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202986.222329:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202986.222330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202986.222332:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559857536:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202986.925944:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202987.949986:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202988.973965:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202989.997984:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788202990.831202:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202990.831218:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202990.831323:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788202990.831354:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788202990.831360:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788202990.831393:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202991.021929:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788202991.341938:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202991.341947:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788202991.341950:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202991.341958:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2122 00000100:00080000:1.0:1788202991.341967:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202991.341973:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788202991.342005:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559857664/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202991.342034:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202991.342037:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788202991.342038:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202991.342041:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2122 00000100:00080000:1.0:1788202991.342044:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202991.342046:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202991.342055:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788202991.342064:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559857792/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788202991.342077:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788202991.342080:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788202991.342082:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788202991.342086:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2122 00000100:00080000:1.0:1788202991.342088:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788202991.342091:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788202991.342094:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788202991.342103:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559857920/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788202991.342115:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559857664:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788202991.342221:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202991.342268:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202991/real 1788202991] req@ffff93c1c5034a80 x1875064559857664/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202991.342288:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559857664/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202991.342303:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202991.342308:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202991.342313:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559857664:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202991.342323:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559857792:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202991.342351:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202991.342366:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202991/real 1788202991] req@ffff93c1c5037100 x1875064559857792/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202991.342381:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559857792/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202991.342393:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202991.342397:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202991.342400:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559857792:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202991.342434:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559857920:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202991.342460:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202991.342485:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202991/real 1788202991] req@ffff93c1c5034e00 x1875064559857920/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202991.342499:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559857920/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202991.342512:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202991.342516:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202991.342520:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559857920:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202992.045978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202993.069985:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202994.093947:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202995.117990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788202995.950145:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202995.950174:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202995.950297:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788202995.950327:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788202995.950333:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788202995.950364:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788202996.141960:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788202996.461981:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202996.461990:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788202996.461994:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202996.462003:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2127 00000100:00080000:2.0:1788202996.462012:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202996.462018:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788202996.462051:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559858048/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202996.462075:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202996.462078:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788202996.462079:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202996.462083:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2127 00000100:00080000:2.0:1788202996.462085:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788202996.462087:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202996.462096:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788202996.462104:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559858176/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788202996.462113:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788202996.462115:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788202996.462117:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788202996.462119:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2127 00000100:00080000:2.0:1788202996.462123:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788202996.462124:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559858048:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788202996.462128:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788202996.462135:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788202996.462143:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559858304/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788202996.462193:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202996.462242:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202996/real 1788202996] req@ffff93c0e62b8000 x1875064559858048/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202996.462261:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559858048/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202996.462269:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202996.462272:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202996.462275:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559858048:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788202996.462283:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559858176:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788202996.462307:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202996.462321:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202996/real 1788202996] req@ffff93c0e62b8380 x1875064559858176/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202996.462334:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559858176/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202996.462343:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202996.462347:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202996.462350:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559858176:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788202996.462371:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559858304:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788202996.462387:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788202996.462398:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788202996/real 1788202996] req@ffff93c0e62b8700 x1875064559858304/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788202996.462411:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559858304/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788202996.462421:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788202996.462425:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788202996.462429:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559858304:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788202997.166064:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202998.189958:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788202999.213941:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203000.237981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203001.069996:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203001.070012:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203001.070102:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203001.070135:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203001.070140:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203001.070175:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203001.261979:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203001.581944:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203001.581952:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203001.581956:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203001.581964:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2132 00000100:00080000:2.0:1788203001.581974:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203001.581980:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203001.582024:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559858432/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203001.582044:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203001.582047:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203001.582048:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203001.582051:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2132 00000100:00080000:2.0:1788203001.582053:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203001.582056:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203001.582075:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203001.582084:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559858560/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203001.582092:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203001.582095:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203001.582096:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203001.582099:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2132 00000100:00080000:2.0:1788203001.582102:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788203001.582107:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559858432:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203001.582107:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203001.582112:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203001.582122:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559858688/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203001.582184:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203001.582236:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203001/real 1788203001] req@ffff93c0e62bb100 x1875064559858432/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203001.582251:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559858432/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203001.582270:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203001.582274:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203001.582278:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559858432:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203001.582288:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559858560:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203001.582310:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203001.582324:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203001/real 1788203001] req@ffff93c0e62bbb80 x1875064559858560/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203001.582333:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559858560/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203001.582339:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203001.582341:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203001.582343:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559858560:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203001.582399:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559858688:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203001.582415:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203001.582423:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203001/real 1788203001] req@ffff93c0e62b9c00 x1875064559858688/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203001.582431:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559858688/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203001.582438:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203001.582440:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203001.582442:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559858688:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203002.285942:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203003.310009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203004.333961:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203005.357958:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203006.189940:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203006.189954:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203006.190033:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203006.190056:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203006.190061:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203006.190092:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203006.381933:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203006.701913:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203006.701921:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203006.701925:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203006.701933:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2137 00000100:00080000:2.0:1788203006.701943:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203006.701950:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203006.701989:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559858816/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203006.702015:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203006.702018:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203006.702020:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203006.702023:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2137 00000100:00080000:2.0:1788203006.702027:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203006.702038:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203006.702048:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203006.702058:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559858944/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203006.702067:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203006.702071:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788203006.702072:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559858816:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203006.702073:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203006.702082:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2137 00000100:00080000:2.0:1788203006.702084:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203006.702087:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203006.702090:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203006.702108:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559859072/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203006.702148:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203006.702190:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203006/real 1788203006] req@ffff93c0e62b9500 x1875064559858816/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203006.702205:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559858816/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203006.702213:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203006.702216:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203006.702219:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559858816:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203006.702237:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559858944:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203006.702252:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203006.702260:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203006/real 1788203006] req@ffff93c0e62bad80 x1875064559858944/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203006.702269:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559858944/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203006.702274:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203006.702276:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203006.702278:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559858944:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203006.702282:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559859072:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203006.702292:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203006.702298:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203006/real 1788203006] req@ffff93c0e62b8a80 x1875064559859072/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203006.702306:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559859072/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203006.702313:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203006.702316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203006.702319:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559859072:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203007.405937:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203008.429936:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203009.453999:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203010.477997:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203011.310012:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203011.310028:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203011.310166:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203011.310194:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203011.310198:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203011.310228:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203011.501931:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203011.822021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203011.822030:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203011.822034:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203011.822043:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2142 00000100:00080000:0.0:1788203011.822052:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203011.822058:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203011.822099:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559859200/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203011.822127:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203011.822130:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203011.822131:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203011.822134:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2142 00000100:00080000:0.0:1788203011.822136:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203011.822139:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203011.822147:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203011.822156:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559859328/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203011.822164:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203011.822167:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203011.822169:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203011.822173:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559859200:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203011.822175:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2142 00000100:00080000:0.0:1788203011.822184:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203011.822187:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203011.822192:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203011.822205:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559859456/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203011.822232:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203011.822276:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203011/real 1788203011] req@ffff93c0e9ce5500 x1875064559859200/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203011.822295:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559859200/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203011.822303:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203011.822306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203011.822313:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559859200:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203011.822336:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559859328:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203011.822391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203011.822405:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203011/real 1788203011] req@ffff93c0e9ce5180 x1875064559859328/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203011.822414:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559859328/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203011.822420:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203011.822422:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203011.822432:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559859328:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203011.822439:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559859456:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203011.822455:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203011.822462:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203011/real 1788203011] req@ffff93c0e9ce7800 x1875064559859456/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203011.822469:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559859456/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203011.822480:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203011.822482:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203011.822484:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559859456:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203012.525993:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203013.549970:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203014.574010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203015.597975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203016.430039:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203016.430053:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203016.430133:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203016.430151:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203016.430154:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203016.430171:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203016.621990:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203016.941986:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203016.941998:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203016.942002:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203016.942012:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2147 00000100:00080000:2.0:1788203016.942032:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203016.942038:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203016.942076:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559859584/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203016.942098:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203016.942101:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203016.942102:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203016.942106:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2147 00000100:00080000:2.0:1788203016.942108:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203016.942111:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203016.942120:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203016.942129:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559859712/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203016.942144:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203016.942148:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1788203016.942148:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559859584:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203016.942151:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203016.942158:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2147 00000100:00080000:2.0:1788203016.942161:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203016.942163:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203016.942167:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203016.942177:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559859840/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203016.942218:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203016.942298:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203016/real 1788203016] req@ffff93c0e62b9f80 x1875064559859584/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203016.942317:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559859584/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203016.942327:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203016.942332:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203016.942335:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559859584:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203016.942360:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559859712:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203016.942383:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203016.942397:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203016/real 1788203016] req@ffff93c0e62bb800 x1875064559859712/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203016.942419:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559859712/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203016.942428:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203016.942432:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203016.942436:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559859712:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203016.942443:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559859840:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203016.942463:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203016.942474:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203016/real 1788203016] req@ffff93c0e62b8a80 x1875064559859840/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203016.942486:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559859840/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203016.942495:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203016.942498:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203016.942501:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559859840:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203017.645924:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203018.670008:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203019.693990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203020.717952:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203021.549984:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203021.550002:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203021.550137:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203021.550172:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203021.550177:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203021.550224:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203021.741988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203022.061940:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203022.061950:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203022.061954:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203022.061966:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2152 00000100:00080000:2.0:1788203022.061979:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203022.061988:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203022.062036:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559859968/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203022.062060:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203022.062063:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203022.062064:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203022.062068:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2152 00000100:00080000:2.0:1788203022.062070:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203022.062072:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203022.062088:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203022.062099:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559860096/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203022.062107:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203022.062110:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203022.062111:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203022.062115:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2152 00000100:00080000:2.0:1788203022.062119:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203022.062123:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203022.062127:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203022.062137:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559860224/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203022.062139:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559859968:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203022.062238:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203022.062274:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203022/real 1788203022] req@ffff93c0e62bad80 x1875064559859968/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203022.062289:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559859968/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203022.062298:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203022.062303:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203022.062307:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559859968:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203022.062317:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559860096:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203022.062342:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203022.062355:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203022/real 1788203022] req@ffff93c0e62b9500 x1875064559860096/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203022.062366:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559860096/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203022.062376:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203022.062379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203022.062382:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559860096:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203022.062412:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559860224:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203022.062436:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203022.062449:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203022/real 1788203022] req@ffff93c0e62b9c00 x1875064559860224/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203022.062463:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559860224/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203022.062474:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203022.062478:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203022.062482:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559860224:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203022.766009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203023.789970:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203024.813940:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203025.837947:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203026.670059:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203026.670078:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203026.670224:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203026.670252:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203026.670256:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203026.670283:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203026.862002:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203027.181959:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203027.181974:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203027.181979:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203027.181993:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2158 00000100:00080000:2.0:1788203027.182008:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203027.182017:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203027.182067:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559860352/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203027.182109:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203027.182112:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203027.182114:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203027.182117:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2158 00000100:00080000:2.0:1788203027.182120:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203027.182123:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203027.182135:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203027.182147:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559860480/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203027.182156:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203027.182160:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203027.182162:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203027.182162:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559860352:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203027.182169:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2158 00000100:00080000:2.0:1788203027.182172:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203027.182175:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203027.182178:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203027.182190:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559860608/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203027.182240:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203027.182281:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203027/real 1788203027] req@ffff93c0e62bbb80 x1875064559860352/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203027.182297:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559860352/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203027.182311:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203027.182314:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203027.182318:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559860352:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203027.182340:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559860480:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203027.182356:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203027.182364:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203027/real 1788203027] req@ffff93c0e62bb100 x1875064559860480/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203027.182373:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559860480/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203027.182379:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203027.182381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203027.182383:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559860480:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203027.182387:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559860608:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203027.182398:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203027.182404:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203027/real 1788203027] req@ffff93c0e62b8700 x1875064559860608/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203027.182411:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559860608/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203027.182419:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203027.182421:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203027.182422:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559860608:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203027.885976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203028.909970:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203029.933926:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203030.957976:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203031.789970:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203031.789986:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203031.790091:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203031.790113:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203031.790117:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203031.790135:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203031.981983:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203032.301935:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203032.301943:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203032.301947:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203032.301955:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2163 00000100:00080000:1.0:1788203032.301964:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203032.301970:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203032.302020:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559860736/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203032.302054:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203032.302059:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203032.302061:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203032.302066:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2163 00000100:00080000:1.0:1788203032.302071:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203032.302075:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203032.302084:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203032.302095:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559860736:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203032.302098:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559860864/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203032.302110:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203032.302113:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203032.302114:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203032.302117:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2163 00000100:00080000:1.0:1788203032.302120:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203032.302123:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203032.302128:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203032.302142:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559860992/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203032.302163:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203032.302205:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203032/real 1788203032] req@ffff93c1c5034e00 x1875064559860736/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203032.302227:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559860736/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203032.302235:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203032.302237:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203032.302240:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559860736:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203032.302259:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559860864:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203032.302275:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203032.302283:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203032/real 1788203032] req@ffff93c1c5037100 x1875064559860864/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203032.302291:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559860864/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203032.302297:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203032.302299:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203032.302302:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559860864:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203032.302305:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559860992:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203032.302317:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203032.302326:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203032/real 1788203032] req@ffff93c1c5034a80 x1875064559860992/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203032.302338:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559860992/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203032.302349:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203032.302352:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203032.302356:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559860992:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203033.005970:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203034.030010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203035.053931:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203036.077978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203036.910073:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203036.910088:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203036.910206:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203036.910242:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203036.910247:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203036.910279:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203037.102017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203037.421960:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203037.421969:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203037.421972:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203037.421980:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2168 00000100:00080000:0.0:1788203037.421989:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203037.421996:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203037.422029:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559861120/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203037.422054:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203037.422059:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203037.422061:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203037.422064:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2168 00000100:00080000:0.0:1788203037.422066:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203037.422069:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203037.422076:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203037.422084:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559861248/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203037.422092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203037.422094:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203037.422096:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203037.422099:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2168 00000100:00080000:0.0:1788203037.422100:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203037.422103:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203037.422105:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203037.422112:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559861376/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203037.422122:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559861120:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203037.422220:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559861248:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203037.422263:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559861376:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203038.126016:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203039.150021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203040.173991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788203040.494046:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:0.0:1788203040.494082:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788203040.494098:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788203040.494147:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788203040.494156:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788203040.494188:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203037/real 1788203040] req@ffff93c0e9ce7100 x1875064559861376/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203040.494209:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559861376/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203040.494222:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203040.494227:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203040.494230:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559861376:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788203040.494237:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203037/real 1788203040] req@ffff93c0e9ce4a80 x1875064559861120/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203040.494248:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559861120/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203040.494262:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203040.494265:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203040.494269:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559861120:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203040.494275:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203037/real 1788203040] req@ffff93c0e9ce5f80 x1875064559861248/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203040.494286:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559861248/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203040.494294:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203040.494297:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203040.494301:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559861248:192.168.201.156@tcp:38:kworker.0 00010000:00010000:0.0:1788203041.197928:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203042.029945:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203042.029956:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203042.030026:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203042.030044:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203042.030046:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203042.030062:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203042.221945:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203042.541999:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203042.542007:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203042.542009:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203042.542017:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2173 00000100:00080000:0.0:1788203042.542024:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203042.542030:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203042.542063:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559861504/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203042.542091:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203042.542094:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203042.542096:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203042.542100:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2173 00000100:00080000:0.0:1788203042.542103:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203042.542107:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203042.542116:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203042.542131:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559861632/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203042.542141:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203042.542144:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203042.542146:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203042.542152:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559861504:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203042.542154:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2173 00000100:00080000:0.0:1788203042.542161:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203042.542165:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203042.542170:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203042.542182:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559861760/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203042.542214:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203042.542259:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203042/real 1788203042] req@ffff93c0e9ce6d80 x1875064559861504/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203042.542277:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559861504/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203042.542287:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203042.542291:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203042.542309:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559861504:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203042.542317:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559861632:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203042.542341:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203042.542354:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203042/real 1788203042] req@ffff93c0e9ce5c00 x1875064559861632/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203042.542366:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559861632/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203042.542374:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203042.542377:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203042.542381:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559861632:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203042.542417:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559861760:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203042.542438:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203042.542449:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203042/real 1788203042] req@ffff93c0e9ce7b80 x1875064559861760/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203042.542461:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559861760/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203042.542470:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203042.542473:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203042.542477:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559861760:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203043.245968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203044.269960:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203045.293977:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203046.317991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203047.150004:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203047.150016:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203047.150099:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203047.150117:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203047.150120:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203047.150135:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203047.341941:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203047.661959:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203047.661969:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203047.661973:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203047.661982:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2178 00000100:00080000:0.0:1788203047.661999:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203047.662007:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203047.662040:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559861888/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203047.662067:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203047.662070:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203047.662071:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203047.662075:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2178 00000100:00080000:0.0:1788203047.662077:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203047.662081:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203047.662089:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203047.662101:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559862016/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203047.662107:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559861888:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203047.662116:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203047.662120:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203047.662122:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203047.662127:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2178 00000100:00080000:0.0:1788203047.662130:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203047.662134:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203047.662139:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203047.662149:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559862144/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203047.662163:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203047.662202:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203047/real 1788203047] req@ffff93c0e9ce6680 x1875064559861888/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203047.662217:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559861888/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203047.662224:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203047.662227:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203047.662229:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559861888:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203047.662250:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559862016:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203047.662263:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203047.662270:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203047/real 1788203047] req@ffff93c0e9ce4000 x1875064559862016/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203047.662277:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559862016/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203047.662282:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203047.662283:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203047.662285:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559862016:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203047.662288:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559862144:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203047.662301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203047.662310:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203047/real 1788203047] req@ffff93c0e9ce6a00 x1875064559862144/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203047.662322:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559862144/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203047.662337:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203047.662340:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203047.662343:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559862144:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203048.365971:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203049.389983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203050.413963:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203051.438002:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203052.270019:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203052.270032:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203052.270112:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203052.270130:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203052.270133:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203052.270157:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203052.461957:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203052.781976:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203052.781988:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203052.781993:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203052.782004:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2183 00000100:00080000:2.0:1788203052.782015:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203052.782023:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203052.782056:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559862272/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203052.782082:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203052.782085:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203052.782086:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203052.782089:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2183 00000100:00080000:2.0:1788203052.782092:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203052.782094:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203052.782102:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203052.782111:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559862400/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203052.782119:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203052.782122:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203052.782123:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203052.782130:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2183 00000100:00100000:3.0:1788203052.782132:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559862272:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203052.782133:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203052.782139:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203052.782144:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203052.782153:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559862528/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203052.782190:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203052.782231:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203052/real 1788203052] req@ffff93c0e62b8380 x1875064559862272/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203052.782246:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559862272/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203052.782254:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203052.782257:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203052.782260:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559862272:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203052.782266:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559862400:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203052.782280:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203052.782289:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203052/real 1788203052] req@ffff93c0e62b8000 x1875064559862400/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203052.782301:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559862400/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203052.782310:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203052.782314:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203052.782317:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559862400:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203052.782343:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559862528:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203052.782363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203052.782372:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203052/real 1788203052] req@ffff93c0e62ba680 x1875064559862528/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203052.782380:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559862528/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203052.782387:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203052.782389:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203052.782392:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559862528:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203053.486005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203054.509958:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203055.533953:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203056.557980:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203057.390015:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203057.390029:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203057.390124:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203057.390143:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203057.390146:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203057.390164:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203057.581977:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203057.901944:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203057.901954:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203057.901958:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203057.901968:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2188 00000100:00080000:2.0:1788203057.901986:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203057.901994:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203057.902034:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559862656/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203057.902067:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203057.902071:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203057.902073:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203057.902077:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2188 00000100:00080000:2.0:1788203057.902081:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203057.902086:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203057.902096:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203057.902108:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559862656:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203057.902113:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559862784/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203057.902134:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203057.902139:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203057.902141:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203057.902146:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2188 00000100:00080000:2.0:1788203057.902160:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788203057.902166:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00080000:2.0:1788203057.902167:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203057.902174:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203057.902192:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559862912/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203057.902215:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203057/real 1788203057] req@ffff93c0e62baa00 x1875064559862656/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203057.902230:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559862656/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203057.902238:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203057.902241:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203057.902245:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559862656:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203057.902262:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559862784:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203057.902277:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203057.902285:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203057/real 1788203057] req@ffff93c0e62bb480 x1875064559862784/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203057.902294:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559862784/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203057.902300:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203057.902302:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203057.902304:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559862784:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203057.902315:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559862912:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203057.902328:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203057.902335:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203057/real 1788203057] req@ffff93c0e62ba680 x1875064559862912/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203057.902342:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559862912/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203057.902349:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203057.902350:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203057.902352:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559862912:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203058.605998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203059.629983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203060.653962:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203061.677968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203062.509985:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203062.509998:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203062.510080:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203062.510098:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203062.510100:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203062.510117:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203062.701954:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203063.021940:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203063.021949:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203063.021954:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203063.021964:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2193 00000100:00080000:2.0:1788203063.021976:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203063.021983:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203063.022021:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559863040/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203063.022056:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203063.022060:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203063.022062:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203063.022067:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2193 00000100:00080000:2.0:1788203063.022070:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203063.022074:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203063.022085:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203063.022101:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559863168/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203063.022114:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203063.022124:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1788203063.022125:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559863040:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203063.022128:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203063.022139:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2193 00000100:00080000:2.0:1788203063.022145:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203063.022149:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203063.022155:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203063.022170:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559863296/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203063.022215:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203063.022262:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203063/real 1788203063] req@ffff93c0e62b8000 x1875064559863040/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203063.022287:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559863040/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203063.022303:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203063.022306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203063.022310:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559863040:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203063.022335:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559863168:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203063.022360:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203063.022370:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203063/real 1788203063] req@ffff93c0e62b8380 x1875064559863168/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203063.022379:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559863168/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203063.022385:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203063.022387:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203063.022390:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559863168:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203063.022394:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559863296:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203063.022407:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203063.022413:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203063/real 1788203063] req@ffff93c0e62b8700 x1875064559863296/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203063.022421:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559863296/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203063.022429:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203063.022431:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203063.022433:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559863296:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203063.725982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203064.749971:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203065.774037:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203066.797974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203067.630005:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203067.630018:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203067.630104:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203067.630122:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203067.630125:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203067.630143:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203067.822014:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203068.141971:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203068.141981:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203068.141985:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203068.141993:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2199 00000100:00080000:2.0:1788203068.142004:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203068.142011:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203068.142050:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559863424/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203068.142081:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203068.142086:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203068.142088:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203068.142093:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2199 00000100:00080000:2.0:1788203068.142097:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203068.142101:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203068.142112:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203068.142128:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559863552/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203068.142150:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203068.142154:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203068.142162:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203068.142164:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559863424:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203068.142174:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2199 00000100:00080000:2.0:1788203068.142181:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203068.142186:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203068.142196:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203068.142214:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559863680/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203068.142255:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203068.142313:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203068/real 1788203068] req@ffff93c0e62bb100 x1875064559863424/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203068.142337:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559863424/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203068.142350:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203068.142355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203068.142363:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559863424:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203068.142406:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559863552:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203068.142428:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203068.142438:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203068/real 1788203068] req@ffff93c0e62bbb80 x1875064559863552/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203068.142447:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559863552/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203068.142454:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203068.142456:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203068.142459:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559863552:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203068.142463:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559863680:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203068.142475:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203068.142482:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203068/real 1788203068] req@ffff93c0e62b9c00 x1875064559863680/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203068.142490:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559863680/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203068.142498:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203068.142500:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203068.142501:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559863680:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203068.846027:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203069.869996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203070.893945:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203071.917944:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203072.749967:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203072.749980:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203072.750061:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203072.750087:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203072.750093:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203072.750116:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203072.942040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203073.261918:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203073.261926:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203073.261930:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203073.261938:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2204 00000100:00080000:2.0:1788203073.261947:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203073.261954:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203073.261985:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559863808/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203073.262014:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203073.262017:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203073.262019:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203073.262023:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2204 00000100:00080000:2.0:1788203073.262025:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203073.262028:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203073.262036:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203073.262047:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559863936/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203073.262055:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203073.262058:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203073.262059:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203073.262062:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2204 00000100:00080000:2.0:1788203073.262064:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203073.262067:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203073.262072:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203073.262072:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559863808:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203073.262083:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559864064/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203073.262139:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203073.262179:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203073/real 1788203073] req@ffff93c0e62b9500 x1875064559863808/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203073.262200:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559863808/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203073.262212:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203073.262216:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203073.262220:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559863808:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203073.262229:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559863936:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203073.262252:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203073.262264:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203073/real 1788203073] req@ffff93c0e62bad80 x1875064559863936/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203073.262301:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559863936/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203073.262317:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203073.262322:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203073.262325:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559863936:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203073.262351:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559864064:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203073.262375:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203073.262387:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203073/real 1788203073] req@ffff93c0e62b8a80 x1875064559864064/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203073.262401:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559864064/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203073.262411:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203073.262415:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203073.262418:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559864064:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203073.965926:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203074.990004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203076.013982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203077.037992:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203077.869993:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203077.870018:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203077.870097:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203077.870115:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203077.870118:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203077.870135:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203078.061974:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203078.381936:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203078.381944:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203078.381948:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203078.381956:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2209 00000100:00080000:2.0:1788203078.381966:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203078.381972:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203078.382005:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559864192/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203078.382029:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203078.382032:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203078.382033:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203078.382037:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2209 00000100:00080000:2.0:1788203078.382039:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203078.382041:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203078.382049:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203078.382057:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559864320/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203078.382065:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203078.382068:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203078.382069:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203078.382072:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2209 00000100:00080000:2.0:1788203078.382074:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203078.382076:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203078.382080:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203078.382087:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559864448/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203078.382094:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559864192:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203078.382161:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203078.382246:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203078/real 1788203078] req@ffff93c0e62bb800 x1875064559864192/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203078.382267:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559864192/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203078.382278:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203078.382283:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203078.382287:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559864192:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203078.382296:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559864320:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203078.382319:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203078.382342:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203078/real 1788203078] req@ffff93c0e62b9f80 x1875064559864320/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203078.382356:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559864320/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203078.382366:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203078.382369:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203078.382372:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559864320:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203078.382397:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559864448:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203078.382427:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203078.382439:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203078/real 1788203078] req@ffff93c0e62b8a80 x1875064559864448/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203078.382451:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559864448/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203078.382461:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203078.382464:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203078.382468:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559864448:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203079.085997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203080.109991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203081.134034:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203082.157952:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203082.990026:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203082.990041:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203082.990151:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203082.990181:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203082.990187:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203082.990221:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203083.181972:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203083.501947:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203083.501956:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203083.501960:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203083.501968:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2214 00000100:00080000:2.0:1788203083.501978:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203083.501983:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203083.502017:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559864576/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203083.502041:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203083.502044:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203083.502045:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203083.502049:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2214 00000100:00080000:2.0:1788203083.502051:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203083.502053:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203083.502061:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203083.502072:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559864704/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203083.502080:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203083.502083:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203083.502084:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203083.502088:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2214 00000100:00080000:2.0:1788203083.502091:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203083.502093:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203083.502097:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203083.502123:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559864832/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203083.502126:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559864576:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203083.502229:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203083.502288:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203083/real 1788203083] req@ffff93c0e62bad80 x1875064559864576/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203083.502314:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559864576/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203083.502327:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203083.502333:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203083.502338:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559864576:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203083.502349:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559864704:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203083.502377:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203083.502392:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203083/real 1788203083] req@ffff93c0e62b9500 x1875064559864704/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203083.502406:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559864704/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203083.502416:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203083.502419:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203083.502423:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559864704:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203083.502457:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559864832:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203083.502480:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203083.502493:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203083/real 1788203083] req@ffff93c0e62b9c00 x1875064559864832/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203083.502505:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559864832/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203083.502523:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203083.502527:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203083.502531:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559864832:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203084.205929:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203085.229996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203086.253944:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203087.277944:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203088.109968:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203088.109980:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203088.110087:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203088.110106:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203088.110109:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203088.110128:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203088.301949:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203088.621901:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203088.621909:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203088.621913:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203088.621921:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2219 00000100:00080000:2.0:1788203088.621931:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203088.621936:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203088.621967:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559864960/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203088.621991:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203088.621996:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203088.621998:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203088.622002:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2219 00000100:00080000:2.0:1788203088.622006:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203088.622010:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203088.622033:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203088.622050:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559864960:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203088.622052:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559865088/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203088.622068:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203088.622072:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203088.622074:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203088.622080:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2219 00000100:00080000:2.0:1788203088.622098:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203088.622101:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000800:00000100:3.0:1788203088.622106:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000008:00100000:2.0:1788203088.622108:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203088.622121:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559865216/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203088.622150:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203088/real 1788203088] req@ffff93c0e62bbb80 x1875064559864960/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203088.622164:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559864960/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203088.622172:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203088.622175:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203088.622177:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559864960:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203088.622195:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559865088:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203088.622210:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203088.622218:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203088/real 1788203088] req@ffff93c0e62bb100 x1875064559865088/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203088.622225:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559865088/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203088.622231:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203088.622233:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203088.622235:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559865088:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203088.622239:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559865216:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203088.622249:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203088.622255:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203088/real 1788203088] req@ffff93c0e62b8700 x1875064559865216/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203088.622262:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559865216/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203088.622269:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203088.622276:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203088.622278:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559865216:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203089.325957:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203090.349977:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203091.373959:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203092.397991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203093.230115:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203093.230128:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203093.230212:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203093.230244:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203093.230249:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203093.230282:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203093.421985:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203093.741989:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203093.741997:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203093.742000:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203093.742008:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2224 00000100:00080000:1.0:1788203093.742017:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203093.742023:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203093.742056:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559865344/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203093.742079:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203093.742082:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203093.742083:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203093.742086:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2224 00000100:00080000:1.0:1788203093.742088:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203093.742090:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203093.742097:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203093.742106:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559865472/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203093.742113:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203093.742116:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203093.742118:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203093.742121:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2224 00000100:00080000:1.0:1788203093.742123:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203093.742127:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203093.742130:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559865344:192.168.201.156@tcp:250:kworker.0 00000008:00100000:1.0:1788203093.742133:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203093.742142:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559865600/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203093.742184:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203093.742216:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203093/real 1788203093] req@ffff93c1c5035c00 x1875064559865344/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203093.742229:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559865344/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203093.742237:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203093.742240:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203093.742243:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559865344:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203093.742247:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559865472:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203093.742261:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203093.742268:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203093/real 1788203093] req@ffff93c1c5037800 x1875064559865472/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203093.742277:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559865472/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203093.742282:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203093.742284:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203093.742286:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559865472:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203093.742303:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559865600:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203093.742314:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203093.742320:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203093/real 1788203093] req@ffff93c1c5036680 x1875064559865600/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203093.742328:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559865600/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203093.742334:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203093.742336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203093.742338:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559865600:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203094.446012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203095.469949:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203096.494043:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203097.518055:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203098.350071:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203098.350084:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203098.350163:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203098.350179:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203098.350182:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203098.350199:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203098.541964:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203098.861992:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203098.862004:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203098.862009:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203098.862020:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2229 00000100:00080000:2.0:1788203098.862032:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203098.862039:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203098.862083:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559865728/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203098.862116:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203098.862120:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203098.862122:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203098.862127:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2229 00000100:00080000:2.0:1788203098.862131:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203098.862136:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203098.862148:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203098.862165:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559865856/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203098.862176:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559865728:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203098.862182:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203098.862189:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203098.862192:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203098.862199:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2229 00000100:00080000:2.0:1788203098.862204:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203098.862210:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203098.862217:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203098.862233:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559865984/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203098.862245:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203098.862288:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203098/real 1788203098] req@ffff93c0e62b8380 x1875064559865728/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203098.862308:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559865728/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203098.862327:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203098.862331:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203098.862336:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559865728:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203098.862364:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559865856:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203098.862392:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203098.862408:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203098/real 1788203098] req@ffff93c0e62b8000 x1875064559865856/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203098.862422:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559865856/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203098.862431:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203098.862434:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203098.862438:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559865856:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203098.862446:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559865984:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203098.862470:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203098.862485:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203098/real 1788203098] req@ffff93c0e62ba680 x1875064559865984/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203098.862498:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559865984/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203098.862509:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203098.862513:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203098.862517:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559865984:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203099.565996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203100.589988:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203101.613975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203102.637972:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203103.470025:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203103.470038:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203103.470128:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203103.470159:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203103.470162:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203103.470186:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203103.661949:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203103.981990:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203103.982001:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203103.982006:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203103.982016:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2234 00000100:00080000:1.0:1788203103.982029:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203103.982037:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203103.982075:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559866112/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203103.982102:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203103.982105:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203103.982106:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203103.982109:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2234 00000100:00080000:1.0:1788203103.982112:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203103.982115:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203103.982122:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203103.982131:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559866240/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203103.982139:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203103.982142:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203103.982143:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203103.982147:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2234 00000100:00100000:3.0:1788203103.982152:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559866112:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788203103.982154:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203103.982164:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203103.982171:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203103.982185:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559866368/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203103.982250:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559866240:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203103.982283:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559866368:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203104.685998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203105.709941:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203106.733990:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788203107.054112:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:0.0:1788203107.054154:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788203107.054167:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788203107.054203:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788203107.054211:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788203107.054256:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203103/real 1788203107] req@ffff93c1c5035c00 x1875064559866368/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203107.054281:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559866368/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203107.054295:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203107.054300:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203107.054305:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559866368:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788203107.054313:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203103/real 1788203107] req@ffff93c1c5036680 x1875064559866112/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203107.054327:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559866112/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203107.054335:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203107.054339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203107.054342:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559866112:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203107.054351:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203103/real 1788203107] req@ffff93c1c5037800 x1875064559866240/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203107.054393:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559866240/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203107.054403:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203107.054407:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203107.054410:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559866240:192.168.201.156@tcp:38:kworker.0 00010000:00010000:0.0:1788203107.758022:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203108.590036:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203108.590051:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203108.590157:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203108.590194:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203108.590199:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203108.590236:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203108.782011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203109.101984:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203109.102001:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203109.102005:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203109.102016:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2239 00000100:00080000:0.0:1788203109.102026:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203109.102034:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203109.102066:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559866496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203109.102097:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203109.102101:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203109.102103:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203109.102107:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2239 00000100:00080000:0.0:1788203109.102112:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203109.102117:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203109.102127:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203109.102130:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559866496:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203109.102145:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559866624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203109.102156:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203109.102161:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203109.102162:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203109.102167:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2239 00000100:00080000:0.0:1788203109.102171:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203109.102175:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203109.102181:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203109.102194:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559866752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203109.102202:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203109.102247:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203109/real 1788203109] req@ffff93c0e9ce4380 x1875064559866496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203109.102272:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559866496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203109.102282:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203109.102286:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203109.102290:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559866496:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203109.102333:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559866624:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203109.102358:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203109.102371:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203109/real 1788203109] req@ffff93c0e9ce4700 x1875064559866624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203109.102382:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559866624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203109.102391:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203109.102394:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203109.102398:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559866624:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203109.102405:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559866752:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203109.102436:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203109.102448:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203109/real 1788203109] req@ffff93c0e9ce7480 x1875064559866752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203109.102459:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559866752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203109.102469:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203109.102472:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203109.102475:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559866752:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203109.806028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203110.829948:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203111.853992:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203112.877936:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203113.709995:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203113.710008:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203113.710089:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203113.710107:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203113.710110:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203113.710135:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203113.901942:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203114.221951:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203114.221963:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203114.221967:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203114.221978:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2245 00000100:00080000:2.0:1788203114.221989:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203114.221997:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203114.222034:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559866880/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203114.222058:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203114.222061:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203114.222064:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203114.222068:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2245 00000100:00080000:2.0:1788203114.222072:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203114.222077:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203114.222087:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203114.222100:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559867008/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203114.222101:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559866880:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203114.222111:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203114.222114:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203114.222116:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203114.222120:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2245 00000100:00080000:2.0:1788203114.222123:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203114.222127:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203114.222132:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203114.222143:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559867136/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203114.222159:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203114.222204:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203114/real 1788203114] req@ffff93c0e62bb480 x1875064559866880/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203114.222223:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559866880/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203114.222235:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203114.222240:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203114.222243:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559866880:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203114.222262:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559867008:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203114.222281:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203114.222298:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203114/real 1788203114] req@ffff93c0e62baa00 x1875064559867008/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203114.222308:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559867008/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203114.222314:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203114.222316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203114.222318:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559867008:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203114.222322:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559867136:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203114.222334:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203114.222341:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203114/real 1788203114] req@ffff93c0e62ba680 x1875064559867136/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203114.222348:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559867136/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203114.222355:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203114.222357:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203114.222358:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559867136:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203114.925993:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203115.949995:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203116.974040:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203117.998009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203118.829978:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203118.830000:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203118.830114:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203118.830134:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203118.830137:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203118.830157:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203119.021907:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203119.341932:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203119.341944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203119.341948:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203119.341960:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2250 00000100:00080000:2.0:1788203119.341972:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203119.341979:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203119.342027:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559867264/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203119.342053:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203119.342055:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203119.342057:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203119.342060:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2250 00000100:00080000:2.0:1788203119.342062:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203119.342065:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203119.342075:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203119.342085:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559867392/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203119.342093:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203119.342095:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203119.342096:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203119.342099:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2250 00000100:00080000:2.0:1788203119.342101:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203119.342104:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203119.342107:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203119.342114:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559867520/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203119.342120:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559867264:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203119.342205:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203119.342262:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203119/real 1788203119] req@ffff93c0e62b8000 x1875064559867264/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203119.342286:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559867264/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203119.342306:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203119.342312:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203119.342317:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559867264:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203119.342326:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559867392:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203119.342353:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203119.342368:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203119/real 1788203119] req@ffff93c0e62b8380 x1875064559867392/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203119.342383:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559867392/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203119.342392:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203119.342396:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203119.342399:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559867392:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203119.342431:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559867520:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203119.342455:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203119.342468:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203119/real 1788203119] req@ffff93c0e62b8700 x1875064559867520/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203119.342482:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559867520/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203119.342513:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203119.342517:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203119.342521:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559867520:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203120.045997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203121.069989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203122.094005:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203123.117925:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203123.950009:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203123.950025:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203123.950122:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203123.950141:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203123.950145:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203123.950164:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203124.142024:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203124.461981:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203124.461992:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203124.461996:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203124.462006:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2255 00000100:00080000:2.0:1788203124.462025:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203124.462032:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203124.462072:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559867648/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203124.462102:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203124.462107:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203124.462109:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203124.462114:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2255 00000100:00080000:2.0:1788203124.462117:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203124.462122:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203124.462132:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203124.462146:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559867648:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203124.462147:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559867776/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203124.462163:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203124.462166:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203124.462169:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203124.462173:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2255 00000100:00080000:2.0:1788203124.462178:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203124.462182:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203124.462187:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203124.462200:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559867904/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203124.462205:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203124.462240:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203124/real 1788203124] req@ffff93c0e62bb100 x1875064559867648/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203124.462255:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559867648/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203124.462263:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203124.462274:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203124.462278:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559867648:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203124.462296:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559867776:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203124.462311:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203124.462319:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203124/real 1788203124] req@ffff93c0e62bbb80 x1875064559867776/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203124.462327:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559867776/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203124.462333:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203124.462335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203124.462338:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559867776:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203124.462341:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559867904:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203124.462352:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203124.462357:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203124/real 1788203124] req@ffff93c0e62b9c00 x1875064559867904/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203124.462365:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559867904/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203124.462372:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203124.462374:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203124.462376:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559867904:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203125.165928:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203126.190027:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203127.214066:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203128.237969:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203129.070014:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203129.070033:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203129.070163:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203129.070184:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203129.070187:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203129.070206:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203129.261977:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203129.581933:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203129.581944:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203129.581949:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203129.581961:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2260 00000100:00080000:2.0:1788203129.581973:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203129.581981:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203129.582027:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559868032/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203129.582054:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203129.582057:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203129.582059:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203129.582064:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2260 00000100:00080000:2.0:1788203129.582068:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203129.582073:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203129.582085:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203129.582109:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559868160/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203129.582110:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559868032:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203129.582142:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203129.582147:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203129.582149:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203129.582155:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2260 00000100:00080000:2.0:1788203129.582160:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203129.582165:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203129.582172:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000800:00000100:3.0:1788203129.582193:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788203129.582201:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559868288/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203129.582236:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203129/real 1788203129] req@ffff93c0e62b9500 x1875064559868032/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203129.582250:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559868032/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203129.582258:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203129.582262:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203129.582265:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559868032:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203129.582289:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559868160:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203129.582348:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203129.582363:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203129/real 1788203129] req@ffff93c0e62bad80 x1875064559868160/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203129.582377:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559868160/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203129.582394:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203129.582398:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203129.582402:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559868160:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203129.582416:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559868288:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203129.582439:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203129.582452:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203129/real 1788203129] req@ffff93c0e62b8a80 x1875064559868288/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203129.582466:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559868288/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203129.582475:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203129.582477:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203129.582480:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559868288:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203130.285922:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203131.309979:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203132.333938:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203133.357987:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203134.189973:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203134.189986:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203134.190066:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203134.190084:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203134.190087:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203134.190104:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203134.381946:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203134.701952:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203134.701961:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203134.701964:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203134.701972:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2265 00000100:00080000:2.0:1788203134.701982:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203134.701988:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203134.702022:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559868416/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203134.702045:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203134.702048:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203134.702049:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203134.702052:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2265 00000100:00080000:2.0:1788203134.702055:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203134.702057:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203134.702063:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203134.702072:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559868544/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203134.702079:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203134.702081:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203134.702083:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203134.702086:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2265 00000100:00080000:2.0:1788203134.702088:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203134.702090:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203134.702092:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203134.702100:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559868672/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203134.702111:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559868416:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203134.702166:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203134.702203:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203134/real 1788203134] req@ffff93c0e62b9f80 x1875064559868416/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203134.702234:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559868416/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203134.702242:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203134.702245:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203134.702249:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559868416:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203134.702257:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559868544:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203134.702282:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203134.702296:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203134/real 1788203134] req@ffff93c0e62bb800 x1875064559868544/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203134.702310:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559868544/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203134.702319:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203134.702322:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203134.702325:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559868544:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203134.702351:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559868672:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203134.702368:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203134.702376:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203134/real 1788203134] req@ffff93c0e62b8a80 x1875064559868672/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203134.702384:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559868672/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203134.702391:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203134.702393:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203134.702395:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559868672:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203135.406012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203136.429959:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203137.454005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203138.477992:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203139.310014:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203139.310029:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203139.310105:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203139.310124:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203139.310127:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203139.310164:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203139.501977:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203139.821910:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203139.821919:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203139.821923:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203139.821933:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2270 00000100:00080000:2.0:1788203139.821945:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203139.821953:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203139.821991:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559868800/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203139.822023:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203139.822032:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203139.822035:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203139.822041:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2270 00000100:00080000:2.0:1788203139.822045:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203139.822050:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203139.822061:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203139.822067:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559868800:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203139.822079:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559868928/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203139.822094:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203139.822098:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203139.822100:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203139.822105:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2270 00000100:00080000:2.0:1788203139.822112:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203139.822117:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000800:00000100:3.0:1788203139.822122:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000008:00100000:2.0:1788203139.822125:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203139.822172:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559869056/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203139.822176:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203139/real 1788203139] req@ffff93c0e62bad80 x1875064559868800/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203139.822206:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559868800/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203139.822214:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203139.822218:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203139.822222:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559868800:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203139.822241:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559868928:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203139.822257:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203139.822266:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203139/real 1788203139] req@ffff93c0e62b9500 x1875064559868928/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203139.822274:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559868928/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203139.822281:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203139.822285:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203139.822288:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559868928:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203139.822295:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559869056:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203139.822315:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203139.822327:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203139/real 1788203139] req@ffff93c0e62b9c00 x1875064559869056/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203139.822340:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559869056/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203139.822355:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203139.822357:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203139.822359:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559869056:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203140.525981:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203141.549992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203142.573971:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203143.598023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203144.430005:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203144.430020:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203144.430130:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203144.430155:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203144.430159:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203144.430190:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203144.621935:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203144.941933:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203144.941942:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203144.941945:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203144.941953:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2275 00000100:00080000:2.0:1788203144.941962:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203144.941968:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203144.942001:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559869184/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203144.942027:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203144.942030:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203144.942031:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203144.942034:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2275 00000100:00080000:2.0:1788203144.942037:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203144.942052:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203144.942060:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203144.942076:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559869312/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203144.942085:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559869184:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203144.942087:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203144.942092:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203144.942094:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203144.942098:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2275 00000100:00080000:2.0:1788203144.942100:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203144.942104:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203144.942109:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203144.942119:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559869440/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203144.942140:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203144.942179:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203144/real 1788203144] req@ffff93c0e62bbb80 x1875064559869184/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203144.942193:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559869184/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203144.942201:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203144.942204:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203144.942207:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559869184:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203144.942225:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559869312:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203144.942240:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203144.942248:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203144/real 1788203144] req@ffff93c0e62bb100 x1875064559869312/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203144.942256:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559869312/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203144.942261:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203144.942263:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203144.942266:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559869312:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203144.942270:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559869440:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203144.942288:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203144.942300:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203144/real 1788203144] req@ffff93c0e62b8700 x1875064559869440/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203144.942313:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559869440/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203144.942323:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203144.942327:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203144.942331:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559869440:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203145.645962:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203146.669995:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203147.693953:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203148.717998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203149.549996:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203149.550010:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203149.550098:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203149.550117:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203149.550120:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203149.550138:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203149.741946:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203150.061934:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203150.061942:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203150.061946:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203150.061965:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2280 00000100:00080000:2.0:1788203150.061975:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203150.061981:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203150.062015:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559869568/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203150.062040:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203150.062044:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203150.062046:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203150.062050:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2280 00000100:00080000:2.0:1788203150.062054:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203150.062059:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203150.062068:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203150.062082:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559869696/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203150.062093:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203150.062094:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559869568:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203150.062097:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203150.062099:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203150.062103:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2280 00000100:00080000:2.0:1788203150.062106:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203150.062110:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203150.062114:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203150.062124:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559869824/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203150.062154:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203150.062205:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203150/real 1788203150] req@ffff93c0e62b8380 x1875064559869568/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203150.062227:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559869568/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203150.062238:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203150.062242:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203150.062246:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559869568:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203150.062274:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559869696:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203150.062302:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203150.062312:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203150/real 1788203150] req@ffff93c0e62b8000 x1875064559869696/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203150.062320:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559869696/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203150.062326:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203150.062328:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203150.062330:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559869696:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203150.062334:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559869824:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203150.062345:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203150.062352:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203150/real 1788203150] req@ffff93c0e62ba680 x1875064559869824/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203150.062359:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559869824/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203150.062366:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203150.062367:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203150.062369:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559869824:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203150.766046:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203151.789959:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203152.814088:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203153.838010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203154.670013:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203154.670027:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203154.670123:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203154.670152:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203154.670162:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203154.670186:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203154.862044:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203155.181987:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203155.181996:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203155.182000:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203155.182009:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2286 00000100:00080000:2.0:1788203155.182019:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203155.182024:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203155.182059:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559869952/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203155.182087:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203155.182091:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203155.182092:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203155.182095:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2286 00000100:00080000:2.0:1788203155.182098:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203155.182100:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203155.182108:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203155.182116:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559870080/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203155.182124:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203155.182127:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203155.182128:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203155.182131:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2286 00000100:00080000:2.0:1788203155.182133:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203155.182136:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203155.182140:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203155.182148:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559870208/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203155.182178:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559869952:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203155.182263:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203155.182311:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203155/real 1788203155] req@ffff93c0e62baa00 x1875064559869952/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203155.182326:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559869952/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203155.182341:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203155.182344:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203155.182348:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559869952:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203155.182353:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559870080:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203155.182370:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203155.182379:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203155/real 1788203155] req@ffff93c0e62bb480 x1875064559870080/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203155.182387:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559870080/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203155.182393:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203155.182395:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203155.182397:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559870080:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203155.182421:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559870208:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203155.182435:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203155.182446:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203155/real 1788203155] req@ffff93c0e62ba680 x1875064559870208/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203155.182459:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559870208/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203155.182471:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203155.182475:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203155.182478:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559870208:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203155.885981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203156.909923:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203157.934013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203158.958009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203159.790038:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203159.790059:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203159.790198:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203159.790221:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203159.790224:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203159.790246:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203159.981956:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203160.301975:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203160.301986:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203160.301990:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203160.302002:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2291 00000100:00080000:2.0:1788203160.302015:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203160.302022:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203160.302073:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559870336/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203160.302102:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203160.302105:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203160.302107:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203160.302111:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2291 00000100:00080000:2.0:1788203160.302116:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203160.302120:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203160.302132:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203160.302143:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559870464/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203160.302151:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203160.302154:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203160.302156:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203160.302162:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2291 00000100:00080000:2.0:1788203160.302165:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203160.302168:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203160.302174:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203160.302187:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559870592/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203160.302201:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559870336:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203160.302272:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203160.302319:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203160/real 1788203160] req@ffff93c0e62b8000 x1875064559870336/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203160.302342:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559870336/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203160.302351:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203160.302355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203160.302358:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559870336:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203160.302364:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559870464:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203160.302379:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203160.302387:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203160/real 1788203160] req@ffff93c0e62b8380 x1875064559870464/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203160.302401:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559870464/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203160.302407:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203160.302409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203160.302412:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559870464:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203160.302438:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559870592:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203160.302461:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203160.302471:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203160/real 1788203160] req@ffff93c0e62b8700 x1875064559870592/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203160.302479:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559870592/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203160.302487:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203160.302489:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203160.302492:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559870592:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203161.005999:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203162.029948:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203163.053988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203164.077984:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203164.910057:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203164.910072:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203164.910168:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203164.910188:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203164.910191:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203164.910209:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203165.102033:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203165.421999:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203165.422011:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203165.422015:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203165.422026:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2296 00000100:00080000:2.0:1788203165.422038:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203165.422046:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203165.422085:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559870720/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203165.422113:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203165.422115:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203165.422117:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203165.422120:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2296 00000100:00080000:2.0:1788203165.422123:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203165.422126:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203165.422135:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203165.422149:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559870848/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203165.422160:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203165.422164:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203165.422166:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203165.422172:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2296 00000100:00080000:2.0:1788203165.422177:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203165.422182:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203165.422188:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203165.422204:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559870976/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203165.422226:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559870720:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203165.422339:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203165.422390:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203165/real 1788203165] req@ffff93c0e62bb100 x1875064559870720/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203165.422406:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559870720/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203165.422416:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203165.422420:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203165.422423:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559870720:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203165.422441:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559870848:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203165.422461:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203165.422470:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203165/real 1788203165] req@ffff93c0e62bbb80 x1875064559870848/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203165.422478:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559870848/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203165.422489:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203165.422491:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203165.422494:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559870848:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203165.422519:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559870976:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203165.422532:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203165.422539:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203165/real 1788203165] req@ffff93c0e62b9c00 x1875064559870976/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203165.422546:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559870976/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203165.422554:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203165.422556:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203165.422558:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559870976:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203166.125997:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203167.150038:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203168.173995:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203169.197998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203170.030038:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203170.030051:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203170.030131:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203170.030152:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203170.030155:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203170.030172:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203170.221991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203170.541978:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203170.541989:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203170.541993:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203170.542004:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2301 00000100:00080000:1.0:1788203170.542016:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203170.542023:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203170.542064:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559871104/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203170.542100:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203170.542103:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203170.542105:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203170.542108:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2301 00000100:00080000:1.0:1788203170.542111:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203170.542114:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203170.542124:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203170.542138:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559871232/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203170.542146:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559871104:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788203170.542157:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203170.542162:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203170.542164:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203170.542170:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2301 00000100:00080000:1.0:1788203170.542175:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203170.542179:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203170.542186:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203170.542201:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559871360/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203170.542256:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559871232:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203170.542279:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559871360:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203171.245941:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203172.270002:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203173.293998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788203173.614143:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:2.0:1788203173.614190:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788203173.614207:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:2.0:1788203173.614249:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:2.0:1788203173.614259:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788203173.614304:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203170/real 1788203173] req@ffff93c1c5037100 x1875064559871232/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203173.614330:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559871232/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203173.614348:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203173.614353:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203173.614358:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559871232:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203173.614367:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203170/real 1788203173] req@ffff93c1c5034e00 x1875064559871360/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203173.614379:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559871360/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203173.614389:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203173.614393:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203173.614397:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559871360:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788203173.614404:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203170/real 1788203173] req@ffff93c1c5034a80 x1875064559871104/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203173.614428:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559871104/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203173.614437:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203173.614441:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203173.614444:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559871104:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788203174.318003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203175.150007:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203175.150024:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203175.150125:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203175.150145:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203175.150148:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203175.150170:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203175.342010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203175.662001:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203175.662009:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203175.662013:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203175.662021:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2306 00000100:00080000:2.0:1788203175.662030:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203175.662036:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203175.662064:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559871488/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203175.662090:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203175.662093:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203175.662095:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203175.662098:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2306 00000100:00080000:2.0:1788203175.662101:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203175.662103:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203175.662111:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203175.662120:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559871616/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203175.662127:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203175.662130:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203175.662131:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203175.662134:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2306 00000100:00080000:2.0:1788203175.662136:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203175.662138:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203175.662141:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203175.662148:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559871488:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203175.662153:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559871744/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203175.662217:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203175.662254:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203175/real 1788203175] req@ffff93c0e62b9500 x1875064559871488/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203175.662274:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559871488/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203175.662285:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203175.662290:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203175.662294:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559871488:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203175.662303:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559871616:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203175.662327:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203175.662340:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203175/real 1788203175] req@ffff93c0e62bad80 x1875064559871616/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203175.662354:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559871616/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203175.662364:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203175.662367:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203175.662371:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559871616:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203175.662415:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559871744:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203175.662440:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203175.662452:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203175/real 1788203175] req@ffff93c0e62b8a80 x1875064559871744/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203175.662465:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559871744/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203175.662483:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203175.662487:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203175.662490:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559871744:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203176.366028:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203177.390004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203178.414003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203179.437971:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203180.269966:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203180.269983:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203180.270065:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203180.270091:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203180.270097:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203180.270131:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203180.462027:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203180.781987:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203180.782000:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203180.782004:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203180.782015:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2311 00000100:00080000:3.0:1788203180.782027:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203180.782035:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203180.782079:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559871872/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203180.782110:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203180.782113:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203180.782114:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203180.782117:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2311 00000100:00080000:3.0:1788203180.782120:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203180.782122:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203180.782131:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203180.782141:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559872000/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203180.782149:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203180.782154:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788203180.782156:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559871872:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203180.782157:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203180.782166:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2311 00000100:00080000:3.0:1788203180.782169:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203180.782171:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203180.782175:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203180.782184:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559872128/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203180.782235:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203180.782288:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203180/real 1788203180] req@ffff93c1c4a4ea00 x1875064559871872/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203180.782313:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559871872/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203180.782326:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203180.782339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203180.782343:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559871872:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203180.782372:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559872000:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203180.782394:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203180.782409:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203180/real 1788203180] req@ffff93c1c4a4c000 x1875064559872000/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203180.782449:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559872000/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203180.782456:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203180.782459:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203180.782461:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559872000:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203180.782466:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559872128:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203180.782485:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203180.782499:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203180/real 1788203180] req@ffff93c1c4a4fb80 x1875064559872128/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203180.782507:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559872128/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203180.782519:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203180.782521:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203180.782524:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559872128:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203181.485959:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203182.509998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203183.533924:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203184.557986:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203185.390071:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203185.390083:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203185.390158:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203185.390183:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203185.390186:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203185.390202:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203185.582031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203185.901903:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203185.901911:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203185.901914:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203185.901921:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2316 00000100:00080000:3.0:1788203185.901929:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203185.901935:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203185.901974:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559872256/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203185.901996:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203185.901998:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203185.901999:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203185.902004:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2316 00000100:00080000:3.0:1788203185.902007:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203185.902009:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203185.902015:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203185.902023:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559872384/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203185.902030:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203185.902032:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203185.902033:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203185.902035:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2316 00000100:00080000:3.0:1788203185.902037:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203185.902039:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203185.902041:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203185.902047:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559872512/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203185.902054:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559872256:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788203185.902114:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203185.902163:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203185/real 1788203185] req@ffff93c1c4a4dc00 x1875064559872256/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203185.902178:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559872256/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203185.902187:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203185.902190:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203185.902196:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559872256:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203185.902203:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559872384:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203185.902227:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203185.902236:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203185/real 1788203185] req@ffff93c1c4a4e680 x1875064559872384/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203185.902243:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559872384/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203185.902249:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203185.902250:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203185.902252:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559872384:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203185.902269:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559872512:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203185.902280:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203185.902286:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203185/real 1788203185] req@ffff93c1c4a4ed80 x1875064559872512/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203185.902296:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559872512/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203185.902302:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203185.902303:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203185.902305:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559872512:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203186.605984:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203187.630024:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203188.653958:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203189.677973:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203190.509979:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203190.509992:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203190.510064:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203190.510093:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203190.510098:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203190.510125:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203190.701949:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203191.021912:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203191.021922:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203191.021926:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203191.021935:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2321 00000100:00080000:0.0:1788203191.021945:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203191.021951:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203191.021986:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559872640/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203191.022012:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203191.022015:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203191.022017:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203191.022022:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2321 00000100:00080000:0.0:1788203191.022026:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203191.022031:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203191.022039:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559872640:192.168.201.156@tcp:250:kworker.0 00000008:00100000:0.0:1788203191.022040:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203191.022058:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559872768/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203191.022069:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203191.022072:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203191.022074:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203191.022078:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2321 00000100:00080000:0.0:1788203191.022082:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203191.022086:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000800:00000100:3.0:1788203191.022090:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000008:00100000:0.0:1788203191.022092:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203191.022106:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559872896/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203191.022121:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203191/real 1788203191] req@ffff93c0e9ce7480 x1875064559872640/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203191.022144:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559872640/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203191.022154:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203191.022158:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203191.022168:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559872640:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203191.022189:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559872768:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203191.022206:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203191.022224:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203191/real 1788203191] req@ffff93c0e9ce4700 x1875064559872768/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203191.022265:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559872768/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203191.022279:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203191.022285:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203191.022288:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559872768:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203191.022295:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559872896:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203191.022313:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203191.022322:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203191/real 1788203191] req@ffff93c0e9ce4380 x1875064559872896/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203191.022334:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559872896/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203191.022356:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203191.022359:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203191.022361:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559872896:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203191.725982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203192.750012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203193.774002:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203194.798013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203195.630023:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203195.630036:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203195.630127:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203195.630153:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203195.630157:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203195.630270:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203195.822020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203196.141939:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203196.141948:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203196.141951:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203196.141960:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2327 00000100:00080000:0.0:1788203196.141970:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203196.141977:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203196.142011:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559873024/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203196.142045:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203196.142049:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203196.142051:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203196.142054:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2327 00000100:00080000:0.0:1788203196.142058:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203196.142062:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203196.142070:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203196.142084:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559873152/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203196.142085:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559873024:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203196.142103:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203196.142107:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203196.142109:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203196.142114:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2327 00000100:00080000:0.0:1788203196.142119:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203196.142123:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203196.142127:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000800:00000100:3.0:1788203196.142138:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203196.142140:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559873280/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203196.142205:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203196/real 1788203196] req@ffff93c0e9ce6a00 x1875064559873024/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203196.142224:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559873024/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203196.142231:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203196.142235:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203196.142237:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559873024:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203196.142256:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559873152:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203196.142272:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203196.142280:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203196/real 1788203196] req@ffff93c0e9ce4000 x1875064559873152/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203196.142288:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559873152/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203196.142296:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203196.142299:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203196.142302:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559873152:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203196.142308:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559873280:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203196.142326:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203196.142336:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203196/real 1788203196] req@ffff93c0e9ce6680 x1875064559873280/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203196.142344:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559873280/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203196.142350:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203196.142352:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203196.142354:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559873280:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203196.846000:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203197.870002:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203198.893956:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203199.918011:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203200.750048:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203200.750065:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203200.750232:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203200.750262:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203200.750266:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203200.750305:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203200.941955:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203201.261959:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203201.261969:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203201.261973:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203201.261983:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2332 00000100:00080000:1.0:1788203201.261994:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203201.262000:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203201.262043:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559873408/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203201.262096:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203201.262099:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203201.262101:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203201.262104:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2332 00000100:00080000:1.0:1788203201.262107:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203201.262113:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203201.262113:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559873408:192.168.201.156@tcp:250:kworker.0 00000008:00100000:1.0:1788203201.262125:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203201.262137:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559873536/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203201.262144:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203201.262147:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203201.262149:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203201.262152:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2332 00000100:00080000:1.0:1788203201.262154:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203201.262156:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203201.262161:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203201.262168:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559873664/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203201.262185:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203201.262222:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203201/real 1788203201] req@ffff93c1c5034a80 x1875064559873408/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203201.262236:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559873408/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203201.262246:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203201.262249:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203201.262252:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559873408:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203201.262272:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559873536:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203201.262289:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203201.262300:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203201/real 1788203201] req@ffff93c1c5034e00 x1875064559873536/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203201.262314:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559873536/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203201.262322:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203201.262325:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203201.262329:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559873536:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203201.262335:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559873664:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203201.262350:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203201.262358:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203201/real 1788203201] req@ffff93c1c5037100 x1875064559873664/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203201.262365:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559873664/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203201.262372:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203201.262374:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203201.262376:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559873664:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203201.965979:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203202.989927:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203204.014006:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203205.037965:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203205.870057:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203205.870076:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203205.870207:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203205.870233:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203205.870237:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203205.870264:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203206.061974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203206.382028:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203206.382038:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203206.382042:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203206.382052:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2337 00000100:00080000:2.0:1788203206.382063:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203206.382070:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203206.382110:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559873792/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203206.382134:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203206.382136:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203206.382137:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203206.382141:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2337 00000100:00080000:2.0:1788203206.382143:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203206.382146:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203206.382155:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203206.382164:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559873920/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203206.382171:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203206.382173:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203206.382174:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203206.382177:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2337 00000100:00080000:2.0:1788203206.382179:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203206.382181:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203206.382185:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203206.382190:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559873792:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203206.382197:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559874048/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203206.382266:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203206.382309:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203206/real 1788203206] req@ffff93c0e62bb800 x1875064559873792/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203206.382324:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559873792/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203206.382336:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203206.382339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203206.382342:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559873792:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203206.382347:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559873920:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203206.382361:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203206.382369:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203206/real 1788203206] req@ffff93c0e62b9f80 x1875064559873920/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203206.382377:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559873920/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203206.382382:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203206.382384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203206.382385:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559873920:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203206.382406:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559874048:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203206.382416:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203206.382421:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203206/real 1788203206] req@ffff93c0e62b8a80 x1875064559874048/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203206.382428:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559874048/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203206.382436:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203206.382438:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203206.382441:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559874048:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203207.085983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203208.109985:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203209.134020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203210.158001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203210.990048:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203210.990070:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203210.990189:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203210.990216:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203210.990221:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203210.990262:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203211.181988:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203211.501976:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203211.501989:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203211.501993:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203211.502007:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2342 00000100:00080000:2.0:1788203211.502022:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203211.502031:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203211.502084:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559874176/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203211.502114:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203211.502117:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203211.502119:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203211.502123:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2342 00000100:00080000:2.0:1788203211.502126:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203211.502129:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203211.502140:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203211.502154:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559874304/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203211.502166:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203211.502170:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1788203211.502170:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559874176:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203211.502174:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203211.502181:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2342 00000100:00080000:2.0:1788203211.502184:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203211.502188:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203211.502193:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203211.502205:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559874432/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203211.502245:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203211.502288:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203211/real 1788203211] req@ffff93c0e62bad80 x1875064559874176/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203211.502303:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559874176/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203211.502312:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203211.502314:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203211.502317:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559874176:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203211.502339:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559874304:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203211.502354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203211.502362:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203211/real 1788203211] req@ffff93c0e62b9500 x1875064559874304/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203211.502373:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559874304/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203211.502381:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203211.502384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203211.502387:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559874304:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203211.502393:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559874432:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203211.502413:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203211.502421:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203211/real 1788203211] req@ffff93c0e62b9c00 x1875064559874432/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203211.502429:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559874432/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203211.502437:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203211.502439:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203211.502441:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559874432:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203212.206048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203213.229989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203214.253931:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203215.277945:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203216.110009:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203216.110024:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203216.110119:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203216.110148:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203216.110162:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203216.110191:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203216.301938:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203216.621993:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203216.622003:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203216.622007:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203216.622018:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2347 00000100:00080000:0.0:1788203216.622028:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203216.622035:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203216.622071:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559874560/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203216.622099:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203216.622102:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203216.622104:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203216.622108:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2347 00000100:00080000:0.0:1788203216.622112:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203216.622115:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203216.622124:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203216.622147:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559874688/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203216.622156:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559874560:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203216.622160:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203216.622165:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203216.622166:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203216.622171:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2347 00000100:00080000:0.0:1788203216.622174:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203216.622178:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203216.622183:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203216.622195:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559874816/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203216.622221:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203216.622267:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203216/real 1788203216] req@ffff93c0e9ce7b80 x1875064559874560/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203216.622285:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559874560/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203216.622295:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203216.622298:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203216.622302:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559874560:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203216.622323:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559874688:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203216.622345:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203216.622356:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203216/real 1788203216] req@ffff93c0e9ce5c00 x1875064559874688/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203216.622368:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559874688/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203216.622382:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203216.622385:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203216.622388:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559874688:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203216.622393:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559874816:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203216.622425:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203216.622436:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203216/real 1788203216] req@ffff93c0e9ce6d80 x1875064559874816/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203216.622446:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559874816/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203216.622455:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203216.622458:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203216.622461:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559874816:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203217.326006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203218.349937:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203219.374064:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203220.397992:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203221.230025:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203221.230036:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203221.230116:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203221.230138:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203221.230141:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203221.230157:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203221.421965:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203221.741997:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203221.742011:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203221.742016:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203221.742031:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2352 00000100:00080000:1.0:1788203221.742044:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203221.742053:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203221.742102:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559874944/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203221.742134:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203221.742138:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203221.742140:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203221.742145:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2352 00000100:00080000:1.0:1788203221.742149:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203221.742154:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203221.742165:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203221.742175:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559874944:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203221.742183:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559875072/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203221.742196:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203221.742200:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203221.742202:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203221.742207:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2352 00000100:00080000:1.0:1788203221.742211:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203221.742215:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203221.742222:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203221.742234:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559875200/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203221.742249:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203221.742309:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203221/real 1788203221] req@ffff93c1c5037800 x1875064559874944/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203221.742331:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559874944/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203221.742345:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203221.742350:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203221.742356:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559874944:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203221.742385:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559875072:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203221.742412:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203221.742424:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203221/real 1788203221] req@ffff93c1c5036680 x1875064559875072/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203221.742437:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559875072/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203221.742445:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203221.742448:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203221.742451:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559875072:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203221.742456:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559875200:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203221.742472:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203221.742480:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203221/real 1788203221] req@ffff93c1c5035c00 x1875064559875200/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203221.742491:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559875200/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203221.742500:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203221.742502:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203221.742505:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559875200:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203222.445939:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203223.469989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203224.493982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203225.517970:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203226.349999:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203226.350017:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203226.350133:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203226.350164:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203226.350169:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203226.350198:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203226.542013:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203226.862015:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203226.862029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203226.862034:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203226.862049:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2357 00000100:00080000:1.0:1788203226.862072:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203226.862084:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203226.862138:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559875328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203226.862187:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203226.862191:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203226.862193:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203226.862197:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2357 00000100:00080000:1.0:1788203226.862201:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203226.862206:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203226.862218:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203226.862232:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559875456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203226.862236:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559875328:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788203226.862255:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203226.862259:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203226.862261:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203226.862267:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2357 00000100:00080000:1.0:1788203226.862271:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203226.862276:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203226.862281:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203226.862293:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559875584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203226.862320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203226.862366:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203226/real 1788203226] req@ffff93c1c5035c00 x1875064559875328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203226.862387:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559875328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203226.862396:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203226.862399:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203226.862402:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559875328:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203226.862422:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559875456:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203226.862438:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203226.862445:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203226/real 1788203226] req@ffff93c1c5036680 x1875064559875456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203226.862453:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559875456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203226.862459:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203226.862461:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203226.862463:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559875456:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203226.862466:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559875584:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203226.862476:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203226.862482:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203226/real 1788203226] req@ffff93c1c5037800 x1875064559875584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203226.862488:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559875584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203226.862498:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203226.862501:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203226.862504:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559875584:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203227.566011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203228.589966:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203229.613936:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203230.638101:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203231.470042:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203231.470053:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203231.470128:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203231.470145:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203231.470148:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203231.470167:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203231.661991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203231.981951:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203231.981958:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203231.981961:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203231.981970:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2362 00000100:00080000:0.0:1788203231.981977:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203231.981982:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203231.982012:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559875712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203231.982034:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203231.982036:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203231.982037:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203231.982040:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2362 00000100:00080000:0.0:1788203231.982041:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203231.982043:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203231.982051:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203231.982061:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559875840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203231.982071:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203231.982074:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203231.982078:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203231.982079:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559875712:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203231.982087:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2362 00000100:00080000:0.0:1788203231.982091:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203231.982096:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203231.982101:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203231.982112:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559875968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203231.982129:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203231.982165:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203231/real 1788203231] req@ffff93c0e9ce5f80 x1875064559875712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203231.982177:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559875712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203231.982184:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203231.982187:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203231.982190:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559875712:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203231.982213:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559875840:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203231.982229:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203231.982236:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203231/real 1788203231] req@ffff93c0e9ce4a80 x1875064559875840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203231.982243:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559875840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203231.982248:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203231.982250:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203231.982252:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559875840:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203231.982255:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559875968:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203231.982264:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203231.982269:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203231/real 1788203231] req@ffff93c0e9ce7100 x1875064559875968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203231.982275:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559875968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203231.982285:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203231.982287:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203231.982288:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559875968:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203232.685967:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203233.709925:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203234.734001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203235.757981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203236.589966:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203236.589981:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203236.590076:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203236.590096:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203236.590099:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203236.590117:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203236.781994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203237.101909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203237.101918:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203237.101921:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203237.101930:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2367 00000100:00080000:0.0:1788203237.101940:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203237.101947:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203237.101981:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559876096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203237.102009:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203237.102013:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203237.102015:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203237.102019:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2367 00000100:00080000:0.0:1788203237.102022:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203237.102026:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203237.102034:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203237.102053:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559876096:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203237.102055:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559876224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203237.102067:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203237.102070:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203237.102071:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203237.102074:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2367 00000100:00080000:0.0:1788203237.102077:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203237.102080:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203237.102084:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203237.102093:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559876352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203237.102151:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559876224:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203237.102187:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559876352:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203237.806020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203238.829995:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203239.853982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788203240.174071:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788203240.174108:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788203240.174120:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788203240.174155:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788203240.174165:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788203240.174209:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203237/real 1788203240] req@ffff93c0e9ce5180 x1875064559876224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203240.174232:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559876224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203240.174242:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203240.174247:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203240.174251:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559876224:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203240.174258:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203237/real 1788203240] req@ffff93c0e9ce5500 x1875064559876352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203240.174269:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559876352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203240.174285:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203240.174288:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203240.174292:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559876352:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788203240.174299:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203237/real 1788203240] req@ffff93c0e9ce7800 x1875064559876096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203240.174309:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559876096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203240.174315:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203240.174319:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203240.174322:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559876096:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788203240.877967:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203241.709995:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203241.710007:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203241.710089:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203241.710106:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203241.710108:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203241.710123:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203241.902043:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203242.221947:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203242.221961:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203242.221966:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203242.221979:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2373 00000100:00080000:0.0:1788203242.221993:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203242.222002:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203242.222046:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559876480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203242.222078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203242.222083:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203242.222084:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203242.222090:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2373 00000100:00080000:0.0:1788203242.222093:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203242.222097:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203242.222108:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203242.222123:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559876608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203242.222133:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203242.222144:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203242.222146:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203242.222147:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559876480:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203242.222153:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2373 00000100:00080000:0.0:1788203242.222160:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203242.222163:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203242.222168:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203242.222183:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559876736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203242.222244:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203242.222307:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203242/real 1788203242] req@ffff93c0e9ce4e00 x1875064559876480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203242.222326:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559876480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203242.222337:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203242.222342:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203242.222346:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559876480:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203242.222412:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559876608:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203242.222437:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203242.222450:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203242/real 1788203242] req@ffff93c0e9ce5880 x1875064559876608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203242.222468:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559876608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203242.222476:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203242.222479:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203242.222482:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559876608:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203242.222488:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559876736:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203242.222507:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203242.222518:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203242/real 1788203242] req@ffff93c0e9ce6300 x1875064559876736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203242.222529:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559876736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203242.222540:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203242.222542:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203242.222545:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559876736:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203242.926032:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203243.950038:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203244.973972:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203245.998031:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203246.830024:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203246.830037:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203246.830121:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203246.830145:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203246.830149:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203246.830173:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203247.021897:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203247.341982:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203247.341997:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203247.342003:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203247.342016:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2378 00000100:00080000:1.0:1788203247.342030:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203247.342039:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203247.342089:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559876864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203247.342123:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203247.342127:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203247.342129:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203247.342133:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2378 00000100:00080000:1.0:1788203247.342137:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203247.342140:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203247.342150:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203247.342161:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559876992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203247.342169:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203247.342173:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1788203247.342174:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559876864:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788203247.342175:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203247.342182:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2378 00000100:00080000:1.0:1788203247.342185:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203247.342188:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203247.342192:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203247.342199:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559877120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203247.342250:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203247.342327:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203247/real 1788203247] req@ffff93c1c5037100 x1875064559876864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203247.342350:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559876864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203247.342363:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203247.342368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203247.342372:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559876864:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203247.342405:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559876992:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203247.342425:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203247.342435:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203247/real 1788203247] req@ffff93c1c5034e00 x1875064559876992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203247.342443:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559876992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203247.342448:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203247.342450:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203247.342452:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559876992:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203247.342457:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559877120:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203247.342468:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203247.342473:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203247/real 1788203247] req@ffff93c1c5034a80 x1875064559877120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203247.342479:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559877120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203247.342486:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203247.342488:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203247.342490:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559877120:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203248.045981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203249.069991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203250.094049:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203251.118015:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203251.949967:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203251.949981:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203251.950088:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203251.950123:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203251.950129:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203251.950166:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203252.141987:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203252.461949:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203252.461957:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203252.461960:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203252.461967:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2383 00000100:00080000:2.0:1788203252.461976:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203252.461981:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203252.462011:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559877248/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203252.462031:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203252.462034:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203252.462035:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203252.462038:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2383 00000100:00080000:2.0:1788203252.462041:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203252.462043:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203252.462052:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203252.462065:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559877376/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203252.462073:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203252.462075:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203252.462077:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203252.462082:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2383 00000100:00100000:3.0:1788203252.462082:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559877248:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203252.462085:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203252.462091:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203252.462095:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203252.462102:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559877504/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203252.462143:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203252.462214:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203252/real 1788203252] req@ffff93c0e62bbb80 x1875064559877248/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203252.462230:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559877248/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203252.462239:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203252.462243:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203252.462246:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559877248:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203252.462277:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559877376:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203252.462301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203252.462313:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203252/real 1788203252] req@ffff93c0e62bb100 x1875064559877376/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203252.462323:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559877376/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203252.462328:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203252.462330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203252.462332:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559877376:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203252.462336:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559877504:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203252.462349:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203252.462356:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203252/real 1788203252] req@ffff93c0e62b8700 x1875064559877504/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203252.462368:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559877504/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203252.462377:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203252.462380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203252.462383:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559877504:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203253.166041:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203254.189952:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203255.213966:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203256.237958:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203257.069952:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203257.069964:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203257.070033:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203257.070048:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203257.070050:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203257.070064:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203257.261952:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203257.581923:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203257.581930:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203257.581933:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203257.581940:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2388 00000100:00080000:2.0:1788203257.581949:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203257.581955:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203257.581990:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559877632/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203257.582016:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203257.582019:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203257.582020:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203257.582023:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2388 00000100:00080000:2.0:1788203257.582026:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203257.582029:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203257.582037:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203257.582047:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559877760/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203257.582050:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559877632:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203257.582063:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203257.582066:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203257.582068:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203257.582071:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2388 00000100:00080000:2.0:1788203257.582073:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203257.582075:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203257.582079:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203257.582099:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559877888/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203257.582113:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203257.582156:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203257/real 1788203257] req@ffff93c0e62b8380 x1875064559877632/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203257.582170:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559877632/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203257.582178:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203257.582180:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203257.582183:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559877632:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203257.582199:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559877760:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203257.582214:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203257.582221:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203257/real 1788203257] req@ffff93c0e62b8000 x1875064559877760/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203257.582229:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559877760/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203257.582237:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203257.582239:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203257.582241:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559877760:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203257.582244:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559877888:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203257.582253:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203257.582258:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203257/real 1788203257] req@ffff93c0e62ba680 x1875064559877888/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203257.582264:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559877888/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203257.582270:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203257.582271:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203257.582273:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559877888:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203258.285981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203259.309994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203260.334353:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203261.357992:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203262.190044:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203262.190057:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203262.190164:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203262.190194:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203262.190199:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203262.190238:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203262.381967:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203262.701945:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203262.701955:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203262.701958:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203262.701968:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2393 00000100:00080000:1.0:1788203262.701979:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203262.701985:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203262.702029:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559878016/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203262.702055:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203262.702059:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203262.702061:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203262.702065:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2393 00000100:00080000:1.0:1788203262.702068:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203262.702072:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203262.702081:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203262.702094:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559878016:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203262.702095:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559878144/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203262.702108:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203262.702112:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203262.702114:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203262.702118:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2393 00000100:00080000:1.0:1788203262.702121:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203262.702124:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203262.702128:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203262.702138:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559878272/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203262.702146:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203262.702183:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203262/real 1788203262] req@ffff93c1c5034a80 x1875064559878016/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203262.702196:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559878016/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203262.702204:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203262.702207:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203262.702209:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559878016:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203262.702225:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559878144:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203262.702237:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203262.702244:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203262/real 1788203262] req@ffff93c1c5034e00 x1875064559878144/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203262.702251:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559878144/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203262.702256:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203262.702258:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203262.702260:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559878144:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203262.702263:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559878272:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203262.702272:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203262.702277:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203262/real 1788203262] req@ffff93c1c5037100 x1875064559878272/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203262.702283:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559878272/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203262.702288:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203262.702290:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203262.702292:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559878272:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203263.406051:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203264.429959:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203265.454094:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203266.477959:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203267.310007:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203267.310026:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203267.310149:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203267.310169:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203267.310172:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203267.310189:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203267.502060:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203267.822003:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203267.822013:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203267.822018:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203267.822031:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2398 00000100:00080000:0.0:1788203267.822044:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203267.822052:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203267.822107:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559878400/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203267.822138:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203267.822141:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203267.822143:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203267.822146:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2398 00000100:00080000:0.0:1788203267.822150:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203267.822154:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203267.822164:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203267.822180:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559878528/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203267.822184:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559878400:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203267.822203:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203267.822208:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203267.822211:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203267.822217:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2398 00000100:00080000:0.0:1788203267.822222:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203267.822226:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203267.822231:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203267.822242:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559878656/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203267.822264:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203267.822316:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203267/real 1788203267] req@ffff93c0e9ce6300 x1875064559878400/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203267.822334:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559878400/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203267.822344:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203267.822349:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203267.822352:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559878400:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203267.822377:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559878528:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203267.822393:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203267.822402:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203267/real 1788203267] req@ffff93c0e9ce5880 x1875064559878528/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203267.822411:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559878528/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203267.822417:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203267.822419:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203267.822421:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559878528:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203267.822425:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559878656:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203267.822437:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203267.822443:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203267/real 1788203267] req@ffff93c0e9ce4e00 x1875064559878656/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203267.822450:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559878656/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203267.822462:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203267.822464:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203267.822466:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559878656:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203268.525955:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203269.550057:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203270.574001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203271.598033:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203272.430009:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203272.430030:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203272.430176:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203272.430200:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203272.430203:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203272.430223:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203272.621983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203272.941984:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203272.941995:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203272.942000:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203272.942013:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2403 00000100:00080000:0.0:1788203272.942025:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203272.942034:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203272.942082:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559878784/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203272.942106:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203272.942110:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203272.942112:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203272.942117:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2403 00000100:00080000:0.0:1788203272.942121:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203272.942125:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203272.942137:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203272.942152:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559878912/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203272.942163:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203272.942164:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559878784:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203272.942166:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203272.942168:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203272.942173:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2403 00000100:00080000:0.0:1788203272.942176:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203272.942180:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203272.942184:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203272.942193:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559879040/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203272.942249:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203272.942305:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203272/real 1788203272] req@ffff93c0e9ce7800 x1875064559878784/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203272.942320:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559878784/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203272.942329:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203272.942332:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203272.942335:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559878784:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203272.942359:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559878912:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203272.942377:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203272.942385:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203272/real 1788203272] req@ffff93c0e9ce5500 x1875064559878912/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203272.942393:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559878912/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203272.942399:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203272.942401:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203272.942403:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559878912:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203272.942409:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559879040:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203272.942421:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203272.942427:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203272/real 1788203272] req@ffff93c0e9ce5180 x1875064559879040/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203272.942434:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559879040/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203272.942442:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203272.942443:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203272.942445:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559879040:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203273.646048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203274.670030:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203275.694077:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203276.718043:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203277.550036:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203277.550049:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203277.550131:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203277.550151:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203277.550156:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203277.550187:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203277.742059:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203278.061925:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203278.061934:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203278.061937:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203278.061946:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2408 00000100:00080000:2.0:1788203278.061955:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203278.061967:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203278.062004:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559879168/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203278.062025:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203278.062028:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203278.062029:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203278.062032:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2408 00000100:00080000:2.0:1788203278.062035:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203278.062037:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203278.062045:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203278.062059:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559879296/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203278.062066:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559879168:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203278.062080:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203278.062084:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203278.062087:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203278.062093:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2408 00000100:00080000:2.0:1788203278.062098:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203278.062103:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203278.062108:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203278.062122:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559879424/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203278.062128:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203278.062165:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203278/real 1788203278] req@ffff93c0e62bb480 x1875064559879168/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203278.062178:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559879168/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203278.062185:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203278.062187:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203278.062190:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559879168:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203278.062209:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559879296:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203278.062230:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203278.062243:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203278/real 1788203278] req@ffff93c0e62baa00 x1875064559879296/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203278.062257:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559879296/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203278.062266:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203278.062270:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203278.062273:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559879296:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203278.062280:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559879424:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203278.062299:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203278.062310:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203278/real 1788203278] req@ffff93c0e62ba680 x1875064559879424/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203278.062322:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559879424/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203278.062332:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203278.062334:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203278.062336:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559879424:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203278.766026:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203279.790030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203280.814021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203281.838024:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203282.669953:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203282.669967:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203282.670045:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203282.670062:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203282.670065:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203282.670082:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203282.862012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203283.181905:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203283.181914:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203283.181918:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203283.181926:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2414 00000100:00080000:2.0:1788203283.181936:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203283.181941:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203283.181975:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559879552/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203283.181999:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203283.182002:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203283.182004:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203283.182007:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2414 00000100:00080000:2.0:1788203283.182009:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203283.182012:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203283.182019:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203283.182027:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559879680/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203283.182035:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203283.182038:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203283.182039:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203283.182042:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2414 00000100:00080000:2.0:1788203283.182044:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203283.182047:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203283.182051:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203283.182057:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559879808/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203283.182064:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559879552:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203283.182126:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203283.182163:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203283/real 1788203283] req@ffff93c0e62b8000 x1875064559879552/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203283.182182:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559879552/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203283.182198:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203283.182202:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203283.182207:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559879552:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203283.182215:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559879680:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203283.182236:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203283.182248:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203283/real 1788203283] req@ffff93c0e62b8380 x1875064559879680/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203283.182260:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559879680/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203283.182269:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203283.182272:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203283.182275:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559879680:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203283.182299:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559879808:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203283.182318:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203283.182329:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203283/real 1788203283] req@ffff93c0e62b8700 x1875064559879808/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203283.182341:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559879808/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203283.182351:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203283.182354:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203283.182358:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559879808:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203283.885980:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203284.910040:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203285.933992:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203286.957955:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203287.790039:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203287.790054:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203287.790144:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203287.790167:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203287.790171:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203287.790188:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203287.982013:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203288.301950:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203288.301961:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203288.301967:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203288.301978:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2419 00000100:00080000:2.0:1788203288.301989:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203288.301997:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203288.302036:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559879936/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203288.302070:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203288.302074:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203288.302076:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203288.302081:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2419 00000100:00080000:2.0:1788203288.302085:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203288.302089:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203288.302100:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203288.302115:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559880064/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203288.302128:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559879936:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203288.302129:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203288.302138:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203288.302140:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203288.302146:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2419 00000100:00080000:2.0:1788203288.302150:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203288.302155:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203288.302161:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203288.302177:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559880192/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203288.302190:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203288.302237:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203288/real 1788203288] req@ffff93c0e62bb100 x1875064559879936/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203288.302256:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559879936/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203288.302264:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203288.302267:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203288.302270:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559879936:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203288.302301:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559880064:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203288.302327:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203288.302340:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203288/real 1788203288] req@ffff93c0e62bbb80 x1875064559880064/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203288.302354:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559880064/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203288.302364:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203288.302368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203288.302371:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559880064:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203288.302377:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559880192:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203288.302394:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203288.302402:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203288/real 1788203288] req@ffff93c0e62b9c00 x1875064559880192/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203288.302410:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559880192/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203288.302418:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203288.302421:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203288.302425:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559880192:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203289.006016:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203290.030044:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203291.053963:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203292.077990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203292.910029:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203292.910050:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203292.910219:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203292.910258:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203292.910264:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203292.910294:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203293.101983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203293.421984:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203293.421999:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203293.422006:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203293.422020:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2424 00000100:00080000:1.0:1788203293.422034:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203293.422044:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203293.422097:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559880320/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203293.422132:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203293.422137:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203293.422139:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203293.422144:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2424 00000100:00080000:1.0:1788203293.422147:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203293.422152:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203293.422166:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203293.422181:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559880320:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203293.422183:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559880448/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203293.422199:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203293.422203:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203293.422206:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203293.422210:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2424 00000100:00080000:1.0:1788203293.422214:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203293.422219:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203293.422226:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203293.422239:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559880576/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203293.422263:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203293.422317:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203293/real 1788203293] req@ffff93c1c5037800 x1875064559880320/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203293.422334:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559880320/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203293.422343:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203293.422346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203293.422350:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559880320:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203293.422388:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559880448:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203293.422407:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203293.422416:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203293/real 1788203293] req@ffff93c1c5036680 x1875064559880448/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203293.422424:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559880448/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203293.422435:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203293.422437:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203293.422439:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559880448:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203293.422443:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559880576:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203293.422454:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203293.422487:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203293/real 1788203293] req@ffff93c1c5035c00 x1875064559880576/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203293.422501:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559880576/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203293.422513:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203293.422517:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203293.422521:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559880576:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203294.126012:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203295.150010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203296.174079:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203297.197950:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203298.030003:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203298.030022:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203298.030157:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203298.030185:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203298.030189:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203298.030212:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203298.221993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203298.541928:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203298.541938:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203298.541942:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203298.541950:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2429 00000100:00080000:1.0:1788203298.541959:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203298.541965:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203298.542001:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559880704/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203298.542029:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203298.542033:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203298.542035:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203298.542040:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2429 00000100:00080000:1.0:1788203298.542044:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203298.542048:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203298.542059:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203298.542076:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559880704:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203298.542077:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559880832/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203298.542101:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203298.542104:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203298.542105:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203298.542109:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2429 00000100:00080000:1.0:1788203298.542112:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203298.542115:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203298.542120:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203298.542130:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559880960/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203298.542143:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203298.542198:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203298/real 1788203298] req@ffff93c1c5035c00 x1875064559880704/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203298.542218:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559880704/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203298.542227:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203298.542230:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203298.542233:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559880704:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203298.542255:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559880832:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203298.542275:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203298.542284:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203298/real 1788203298] req@ffff93c1c5036680 x1875064559880832/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203298.542292:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559880832/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203298.542298:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203298.542300:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203298.542302:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559880832:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203298.542307:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559880960:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203298.542317:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203298.542323:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203298/real 1788203298] req@ffff93c1c5037800 x1875064559880960/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203298.542330:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559880960/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203298.542337:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203298.542339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203298.542340:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559880960:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203299.246001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203300.269978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203301.293989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203302.318071:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203303.149984:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203303.149999:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203303.150086:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203303.150108:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203303.150111:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203303.150128:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203303.341971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203303.661955:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203303.661964:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203303.661968:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203303.661976:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2434 00000100:00080000:1.0:1788203303.661986:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203303.661992:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203303.662025:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559881088/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203303.662054:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203303.662056:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203303.662058:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203303.662061:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2434 00000100:00080000:1.0:1788203303.662063:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203303.662065:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203303.662072:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203303.662081:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559881216/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203303.662088:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203303.662091:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203303.662092:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203303.662096:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2434 00000100:00080000:1.0:1788203303.662098:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203303.662100:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203303.662103:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203303.662109:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559881344/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203303.662120:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559881088:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203303.662233:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559881216:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203303.662279:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559881344:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203304.366040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203305.390012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203306.414119:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788203306.734207:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788203306.734300:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788203306.734324:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788203306.734383:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788203306.734395:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788203306.734664:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203303/real 1788203306] req@ffff93c1c5034a80 x1875064559881344/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203306.734695:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559881344/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203306.734711:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203306.734717:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203306.734722:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559881344:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788203306.734732:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203303/real 1788203306] req@ffff93c1c5037100 x1875064559881088/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203306.734747:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559881088/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203306.734756:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203306.734760:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203306.734764:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559881088:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203306.734773:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203303/real 1788203306] req@ffff93c1c5034e00 x1875064559881216/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203306.734787:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559881216/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203306.734797:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203306.734800:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203306.734803:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559881216:192.168.201.156@tcp:38:kworker.0 00010000:00010000:0.0:1788203307.437945:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203308.270008:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203308.270021:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203308.270102:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203308.270123:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203308.270125:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203308.270143:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203308.462060:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203308.781924:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203308.781932:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203308.781935:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203308.781943:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2439 00000100:00080000:1.0:1788203308.781959:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203308.781966:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203308.781998:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559881472/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203308.782014:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203308.782017:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203308.782019:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203308.782022:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2439 00000100:00080000:1.0:1788203308.782025:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203308.782028:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203308.782036:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203308.782046:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559881472:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203308.782047:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559881600/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203308.782062:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203308.782064:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203308.782065:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203308.782068:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2439 00000100:00080000:1.0:1788203308.782069:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203308.782072:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203308.782076:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203308.782085:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559881728/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203308.782119:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203308.782158:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203308/real 1788203308] req@ffff93c1c5034e00 x1875064559881472/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203308.782176:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559881472/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203308.782187:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203308.782192:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203308.782211:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559881472:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203308.782236:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559881600:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203308.782262:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203308.782275:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203308/real 1788203308] req@ffff93c1c5037100 x1875064559881600/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203308.782289:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559881600/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203308.782299:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203308.782303:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203308.782307:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559881600:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203308.782314:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559881728:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203308.782334:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203308.782345:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203308/real 1788203308] req@ffff93c1c5034a80 x1875064559881728/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203308.782366:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559881728/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203308.782377:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203308.782381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203308.782385:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559881728:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203309.486001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203310.510044:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203311.534024:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203312.558052:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203313.390008:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203313.390032:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203313.390215:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203313.390252:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203313.390258:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203313.390294:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203313.581947:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203313.901938:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203313.901950:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203313.901954:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203313.901966:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2444 00000100:00080000:1.0:1788203313.901978:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203313.901986:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203313.902032:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559881856/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203313.902060:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203313.902063:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203313.902064:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203313.902068:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2444 00000100:00080000:1.0:1788203313.902070:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203313.902073:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203313.902083:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203313.902094:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559881984/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203313.902102:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203313.902125:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203313.902127:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203313.902131:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2444 00000100:00080000:1.0:1788203313.902134:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203313.902137:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203313.902143:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203313.902153:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559882112/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203313.902158:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559881856:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788203313.902229:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203313.902277:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203313/real 1788203313] req@ffff93c1c5037800 x1875064559881856/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203313.902293:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559881856/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203313.902303:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203313.902307:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203313.902310:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559881856:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203313.902316:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559881984:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203313.902332:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203313.902340:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203313/real 1788203313] req@ffff93c1c5036680 x1875064559881984/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203313.902353:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559881984/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203313.902359:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203313.902361:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203313.902364:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559881984:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203313.902383:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559882112:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203313.902399:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203313.902410:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203313/real 1788203313] req@ffff93c1c5035c00 x1875064559882112/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203313.902423:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559882112/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203313.902435:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203313.902439:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203313.902443:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559882112:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203314.606036:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203315.629942:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203316.654107:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203317.677966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203318.510000:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203318.510014:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203318.510113:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203318.510138:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203318.510141:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203318.510168:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203318.702009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203319.021943:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203319.021951:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203319.021955:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203319.021962:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2449 00000100:00080000:2.0:1788203319.021971:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203319.021977:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203319.022006:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559882240/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203319.022046:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203319.022049:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203319.022051:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203319.022055:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2449 00000100:00080000:2.0:1788203319.022058:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203319.022062:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203319.022071:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203319.022072:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559882240:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203319.022085:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559882368/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203319.022094:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203319.022097:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203319.022099:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203319.022102:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2449 00000100:00080000:2.0:1788203319.022105:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203319.022107:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203319.022111:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203319.022118:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559882496/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203319.022133:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203319.022183:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203319/real 1788203319] req@ffff93c0e62b9500 x1875064559882240/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203319.022199:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559882240/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203319.022208:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203319.022210:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203319.022214:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559882240:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203319.022233:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559882368:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203319.022253:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203319.022267:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203319/real 1788203319] req@ffff93c0e62bad80 x1875064559882368/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203319.022281:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559882368/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203319.022290:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203319.022294:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203319.022297:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559882368:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203319.022304:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559882496:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203319.022325:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203319.022335:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203319/real 1788203319] req@ffff93c0e62b8a80 x1875064559882496/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203319.022343:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559882496/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203319.022352:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203319.022355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203319.022359:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559882496:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203319.725998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203320.750021:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203321.774074:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203322.798001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203323.630035:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203323.630052:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203323.630175:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203323.630212:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203323.630218:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203323.630254:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203323.822064:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203324.141980:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203324.141989:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203324.141992:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203324.142002:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2455 00000100:00080000:2.0:1788203324.142011:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203324.142017:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203324.142054:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559882624/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203324.142079:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203324.142082:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203324.142084:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203324.142087:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2455 00000100:00080000:2.0:1788203324.142089:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203324.142092:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203324.142099:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203324.142108:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559882752/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203324.142115:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203324.142118:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203324.142120:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203324.142123:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2455 00000100:00080000:2.0:1788203324.142125:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203324.142129:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203324.142134:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203324.142142:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559882880/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203324.142178:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559882624:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788203324.142268:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203324.142333:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203324/real 1788203324] req@ffff93c0e62b9f80 x1875064559882624/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203324.142352:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559882624/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203324.142368:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203324.142372:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203324.142376:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559882624:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203324.142382:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559882752:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203324.142402:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203324.142412:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203324/real 1788203324] req@ffff93c0e62bb800 x1875064559882752/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203324.142421:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559882752/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203324.142427:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203324.142430:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203324.142432:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559882752:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203324.142436:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559882880:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203324.142449:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203324.142456:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203324/real 1788203324] req@ffff93c0e62b8a80 x1875064559882880/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203324.142464:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559882880/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203324.142472:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203324.142474:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203324.142475:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559882880:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203324.846047:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203325.870014:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203326.894028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203327.918011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203328.750057:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203328.750071:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203328.750159:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203328.750186:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203328.750191:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203328.750224:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203328.942072:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203329.261957:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203329.261968:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203329.261973:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203329.261984:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2460 00000100:00080000:2.0:1788203329.261995:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203329.262002:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203329.262037:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559883008/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203329.262065:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203329.262068:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203329.262069:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203329.262073:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2460 00000100:00080000:2.0:1788203329.262076:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203329.262079:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203329.262087:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203329.262097:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559883136/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203329.262104:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203329.262107:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203329.262108:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203329.262111:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2460 00000100:00080000:2.0:1788203329.262113:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203329.262118:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203329.262120:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559883008:192.168.201.156@tcp:250:kworker.0 00000008:00100000:2.0:1788203329.262122:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203329.262132:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559883264/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203329.262191:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203329.262232:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203329/real 1788203329] req@ffff93c0e62bad80 x1875064559883008/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203329.262253:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559883008/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203329.262262:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203329.262266:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203329.262270:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559883008:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203329.262278:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559883136:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203329.262301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203329.262315:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203329/real 1788203329] req@ffff93c0e62b9500 x1875064559883136/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203329.262328:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559883136/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203329.262336:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203329.262339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203329.262342:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559883136:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203329.262365:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559883264:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203329.262387:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203329.262400:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203329/real 1788203329] req@ffff93c0e62b9c00 x1875064559883264/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203329.262412:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559883264/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203329.262423:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203329.262426:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203329.262429:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559883264:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203329.965975:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203330.990066:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203332.014014:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203333.038051:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203333.870108:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203333.870130:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203333.870277:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203333.870324:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203333.870329:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203333.870360:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203334.062038:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203334.381988:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203334.382004:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203334.382010:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203334.382025:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2465 00000100:00080000:1.0:1788203334.382039:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203334.382047:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203334.382100:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559883392/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203334.382133:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203334.382138:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203334.382140:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203334.382144:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2465 00000100:00080000:1.0:1788203334.382148:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203334.382154:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203334.382166:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203334.382179:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559883392:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203334.382187:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559883520/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203334.382200:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203334.382204:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203334.382205:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203334.382209:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2465 00000100:00080000:1.0:1788203334.382212:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203334.382215:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203334.382220:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203334.382231:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559883648/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203334.382268:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203334.382333:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203334/real 1788203334] req@ffff93c1c5035c00 x1875064559883392/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203334.382375:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559883392/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203334.382389:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203334.382394:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203334.382397:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559883392:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203334.382423:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559883520:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203334.382442:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203334.382452:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203334/real 1788203334] req@ffff93c1c5036680 x1875064559883520/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203334.382461:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559883520/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203334.382467:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203334.382469:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203334.382472:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559883520:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203334.382476:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559883648:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203334.382487:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203334.382494:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203334/real 1788203334] req@ffff93c1c5037800 x1875064559883648/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203334.382502:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559883648/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203334.382514:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203334.382516:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203334.382519:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559883648:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203335.086038:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203336.110023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203337.134980:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203338.158246:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203338.990047:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203338.990061:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203338.990150:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203338.990170:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203338.990173:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203338.990191:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203339.181966:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203339.501981:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203339.501989:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203339.501993:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203339.502002:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2470 00000100:00080000:0.0:1788203339.502011:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203339.502018:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203339.502052:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559883776/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203339.502074:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203339.502076:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203339.502077:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203339.502080:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2470 00000100:00080000:0.0:1788203339.502083:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203339.502086:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203339.502094:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203339.502107:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559883904/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203339.502118:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203339.502122:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203339.502124:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203339.502131:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2470 00000100:00100000:3.0:1788203339.502132:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559883776:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203339.502138:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203339.502144:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203339.502150:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203339.502164:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559884032/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203339.502196:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203339.502240:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203339/real 1788203339] req@ffff93c0e9ce7100 x1875064559883776/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203339.502270:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559883776/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203339.502279:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203339.502282:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203339.502285:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559883776:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203339.502290:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559883904:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203339.502309:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203339.502318:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203339/real 1788203339] req@ffff93c0e9ce4a80 x1875064559883904/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203339.502326:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559883904/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203339.502332:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203339.502334:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203339.502336:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559883904:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203339.502355:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559884032:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203339.502367:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203339.502373:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203339/real 1788203339] req@ffff93c0e9ce5f80 x1875064559884032/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203339.502381:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559884032/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203339.502387:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203339.502389:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203339.502391:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559884032:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203340.205986:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203341.230042:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203342.253963:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203343.278115:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203344.110047:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203344.110067:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203344.110186:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203344.110209:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203344.110213:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203344.110231:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203344.301999:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203344.622010:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203344.622025:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203344.622031:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203344.622044:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2475 00000100:00080000:3.0:1788203344.622058:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203344.622066:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203344.622114:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4df80 x1875064559884160/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203344.622142:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203344.622145:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203344.622147:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203344.622150:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2475 00000100:00080000:3.0:1788203344.622153:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203344.622156:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203344.622167:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203344.622177:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f100 x1875064559884288/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203344.622185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203344.622189:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203344.622191:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788203344.622194:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559884160:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203344.622196:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2475 00000100:00080000:3.0:1788203344.622202:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203344.622206:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203344.622209:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203344.622219:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559884416/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203344.622274:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203344.622332:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203344/real 1788203344] req@ffff93c1c4a4df80 x1875064559884160/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203344.622356:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4df80 x1875064559884160/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203344.622369:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203344.622376:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203344.622382:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559884160:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203344.622411:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559884288:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203344.622432:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203344.622443:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203344/real 1788203344] req@ffff93c1c4a4f100 x1875064559884288/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203344.622452:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f100 x1875064559884288/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203344.622458:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203344.622461:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203344.622464:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559884288:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203344.622468:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559884416:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203344.622479:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203344.622486:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203344/real 1788203344] req@ffff93c1c4a4ed80 x1875064559884416/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203344.622494:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559884416/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203344.622502:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203344.622504:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203344.622506:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559884416:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203345.326093:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203346.349942:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203347.374093:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203348.398020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203349.230019:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203349.230032:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203349.230119:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203349.230149:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203349.230154:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203349.230185:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203349.422067:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203349.741935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203349.741944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203349.741948:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203349.741955:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2480 00000100:00080000:0.0:1788203349.741965:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203349.741971:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203349.742006:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559884544/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203349.742031:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203349.742034:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203349.742035:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203349.742038:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2480 00000100:00080000:0.0:1788203349.742041:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203349.742044:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203349.742053:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203349.742063:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559884544:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203349.742069:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559884672/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203349.742083:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203349.742088:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203349.742090:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203349.742096:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2480 00000100:00080000:0.0:1788203349.742101:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203349.742105:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203349.742111:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000800:00000100:3.0:1788203349.742120:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203349.742124:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559884800/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203349.742166:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203349/real 1788203349] req@ffff93c0e9ce6d80 x1875064559884544/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203349.742180:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559884544/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203349.742188:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203349.742191:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203349.742196:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559884544:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203349.742224:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559884672:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203349.742244:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203349.742257:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203349/real 1788203349] req@ffff93c0e9ce5c00 x1875064559884672/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203349.742268:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559884672/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203349.742275:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203349.742278:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203349.742281:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559884672:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203349.742286:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559884800:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203349.742304:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203349.742315:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203349/real 1788203349] req@ffff93c0e9ce7b80 x1875064559884800/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203349.742326:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559884800/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203349.742341:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203349.742344:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203349.742346:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559884800:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203350.446028:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203351.470092:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203352.493989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203353.518035:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203354.350050:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203354.350065:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203354.350170:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203354.350202:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203354.350207:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203354.350241:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203354.541944:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203354.861996:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203354.862007:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203354.862012:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203354.862024:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2485 00000100:00080000:2.0:1788203354.862035:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203354.862043:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203354.862082:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559884928/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203354.862106:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203354.862109:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203354.862110:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203354.862113:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2485 00000100:00080000:2.0:1788203354.862117:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203354.862119:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203354.862127:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203354.862136:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559885056/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203354.862144:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203354.862147:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203354.862149:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203354.862154:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2485 00000100:00100000:3.0:1788203354.862156:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559884928:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203354.862157:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203354.862162:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203354.862169:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203354.862177:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559885184/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203354.862225:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203354.862273:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203354/real 1788203354] req@ffff93c0e62bbb80 x1875064559884928/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203354.862292:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559884928/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203354.862303:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203354.862307:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203354.862311:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559884928:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203354.862334:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559885056:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203354.862358:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203354.862370:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203354/real 1788203354] req@ffff93c0e62bb100 x1875064559885056/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203354.862383:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559885056/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203354.862392:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203354.862409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203354.862413:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559885056:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203354.862421:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559885184:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203354.862443:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203354.862455:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203354/real 1788203354] req@ffff93c0e62b8700 x1875064559885184/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203354.862467:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559885184/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203354.862478:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203354.862482:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203354.862485:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559885184:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203355.566028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203356.589925:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203357.614068:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203358.638005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203359.470034:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203359.470052:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203359.470195:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203359.470221:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203359.470224:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203359.470250:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203359.662042:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203359.981908:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203359.981916:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203359.981920:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203359.981928:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2490 00000100:00080000:2.0:1788203359.981937:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203359.981944:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203359.981983:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559885312/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203359.982013:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203359.982017:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203359.982018:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203359.982022:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2490 00000100:00080000:2.0:1788203359.982025:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203359.982029:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203359.982037:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203359.982049:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559885440/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203359.982059:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203359.982060:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559885312:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203359.982065:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203359.982069:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203359.982073:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2490 00000100:00080000:2.0:1788203359.982076:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203359.982078:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203359.982082:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203359.982090:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559885568/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203359.982124:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203359.982164:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203359/real 1788203359] req@ffff93c0e62b8380 x1875064559885312/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203359.982180:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559885312/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203359.982188:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203359.982191:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203359.982194:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559885312:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203359.982213:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559885440:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203359.982228:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203359.982236:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203359/real 1788203359] req@ffff93c0e62b8000 x1875064559885440/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203359.982245:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559885440/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203359.982251:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203359.982253:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203359.982255:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559885440:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203359.982258:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559885568:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203359.982269:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203359.982275:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203359/real 1788203359] req@ffff93c0e62ba680 x1875064559885568/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203359.982283:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559885568/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203359.982289:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203359.982291:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203359.982293:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559885568:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203360.686014:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203361.710002:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203362.734004:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203363.758018:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203364.590013:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203364.590030:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203364.590135:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203364.590163:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203364.590168:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203364.590201:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203364.782001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203365.101960:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203365.101971:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203365.101975:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203365.101983:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2495 00000100:00080000:2.0:1788203365.101993:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203365.101999:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203365.102034:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559885696/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203365.102059:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203365.102061:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203365.102066:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203365.102069:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2495 00000100:00080000:2.0:1788203365.102072:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203365.102075:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203365.102082:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203365.102092:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559885824/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203365.102100:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203365.102102:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203365.102104:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203365.102107:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2495 00000100:00080000:2.0:1788203365.102111:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788203365.102112:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559885696:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203365.102116:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203365.102123:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203365.102131:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559885952/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203365.102177:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203365.102228:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203365/real 1788203365] req@ffff93c0e62baa00 x1875064559885696/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203365.102249:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559885696/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203365.102267:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203365.102270:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203365.102273:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559885696:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203365.102279:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559885824:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203365.102299:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203365.102308:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203365/real 1788203365] req@ffff93c0e62bb480 x1875064559885824/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203365.102317:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559885824/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203365.102322:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203365.102325:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203365.102327:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559885824:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203365.102345:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559885952:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203365.102357:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203365.102363:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203365/real 1788203365] req@ffff93c0e62ba680 x1875064559885952/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203365.102371:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559885952/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203365.102377:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203365.102379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203365.102381:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559885952:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203365.806043:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203366.830059:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203367.854031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203368.878057:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203369.709995:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203369.710009:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203369.710092:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203369.710112:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203369.710115:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203369.710133:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203369.901987:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203370.221986:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203370.221996:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203370.222000:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203370.222011:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2501 00000100:00080000:1.0:1788203370.222023:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203370.222031:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203370.222070:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559886080/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203370.222099:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203370.222104:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203370.222106:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203370.222110:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2501 00000100:00080000:1.0:1788203370.222113:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203370.222118:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203370.222142:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203370.222158:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559886208/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203370.222169:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203370.222174:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203370.222177:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203370.222182:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2501 00000100:00080000:1.0:1788203370.222186:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203370.222198:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203370.222200:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559886080:192.168.201.156@tcp:250:kworker.0 00000008:00100000:1.0:1788203370.222209:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203370.222225:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559886336/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203370.222361:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559886208:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203370.222408:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559886336:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203370.926035:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203371.950074:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203372.973985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788203373.294101:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:0.0:1788203373.294146:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788203373.294164:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788203373.294204:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788203373.294215:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788203373.294268:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203370/real 1788203373] req@ffff93c1c5034e00 x1875064559886336/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203373.294288:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559886336/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203373.294298:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203373.294302:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203373.294313:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559886336:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788203373.294322:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203370/real 1788203373] req@ffff93c1c5034a80 x1875064559886080/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203373.294335:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559886080/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203373.294344:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203373.294348:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203373.294351:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559886080:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203373.294358:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203370/real 1788203373] req@ffff93c1c5037100 x1875064559886208/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203373.294371:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559886208/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203373.294379:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203373.294382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203373.294385:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559886208:192.168.201.156@tcp:38:kworker.0 00010000:00010000:2.0:1788203373.997995:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203374.830039:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203374.830066:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203374.830149:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203374.830170:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203374.830173:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203374.830190:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203375.021925:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203375.341954:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203375.341963:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203375.341967:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203375.341976:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2506 00000100:00080000:1.0:1788203375.341985:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203375.341991:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203375.342021:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559886464/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203375.342059:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203375.342062:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203375.342064:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203375.342067:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2506 00000100:00080000:1.0:1788203375.342070:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203375.342074:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203375.342084:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203375.342101:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559886592/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203375.342115:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203375.342120:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1788203375.342122:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559886464:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788203375.342123:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203375.342133:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2506 00000100:00080000:1.0:1788203375.342136:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203375.342139:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203375.342144:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203375.342155:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559886720/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203375.342196:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203375.342254:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203375/real 1788203375] req@ffff93c1c5037100 x1875064559886464/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203375.342275:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559886464/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203375.342286:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203375.342290:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203375.342294:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559886464:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203375.342318:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559886592:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203375.342344:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203375.342357:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203375/real 1788203375] req@ffff93c1c5034a80 x1875064559886592/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203375.342370:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559886592/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203375.342379:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203375.342392:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203375.342396:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559886592:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203375.342403:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559886720:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203375.342425:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203375.342438:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203375/real 1788203375] req@ffff93c1c5034e00 x1875064559886720/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203375.342450:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559886720/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203375.342460:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203375.342463:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203375.342467:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559886720:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203376.045981:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203377.069991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203378.094014:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203379.118011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203379.950007:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203379.950029:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203379.950173:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203379.950199:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203379.950203:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203379.950224:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203380.142110:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203380.461947:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203380.461995:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203380.461999:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203380.462018:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2511 00000100:00080000:1.0:1788203380.462035:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203380.462043:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203380.462089:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559886848/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203380.462119:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203380.462123:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203380.462126:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203380.462131:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2511 00000100:00080000:1.0:1788203380.462134:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203380.462139:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203380.462151:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203380.462166:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559886976/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203380.462177:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203380.462181:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203380.462183:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203380.462183:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559886848:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788203380.462187:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2511 00000100:00080000:1.0:1788203380.462191:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203380.462194:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203380.462213:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203380.462225:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559887104/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203380.462268:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203380.462321:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203380/real 1788203380] req@ffff93c1c5037800 x1875064559886848/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203380.462338:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559886848/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203380.462348:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203380.462352:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203380.462355:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559886848:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203380.462382:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559886976:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203380.462409:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203380.462424:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203380/real 1788203380] req@ffff93c1c5036680 x1875064559886976/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203380.462436:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559886976/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203380.462442:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203380.462445:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203380.462447:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559886976:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203380.462452:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559887104:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203380.462468:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203380.462476:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203380/real 1788203380] req@ffff93c1c5035c00 x1875064559887104/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203380.462484:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559887104/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203380.462491:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203380.462493:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203380.462495:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559887104:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203381.166012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203382.190047:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203383.214056:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203384.238052:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203385.070031:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203385.070045:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203385.070131:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203385.070153:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203385.070155:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203385.070174:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203385.262003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203385.581999:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203385.582010:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203385.582014:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203385.582022:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2516 00000100:00080000:1.0:1788203385.582033:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203385.582039:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203385.582073:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559887232/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203385.582099:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203385.582102:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203385.582103:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203385.582106:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2516 00000100:00080000:1.0:1788203385.582108:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203385.582110:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203385.582118:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203385.582126:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559887360/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203385.582133:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203385.582136:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203385.582137:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203385.582141:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2516 00000100:00080000:1.0:1788203385.582143:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203385.582145:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203385.582150:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203385.582156:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559887488/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203385.582162:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559887232:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203385.582226:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203385.582273:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203385/real 1788203385] req@ffff93c1c5035c00 x1875064559887232/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203385.582293:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559887232/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203385.582310:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203385.582315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203385.582319:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559887232:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203385.582326:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559887360:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203385.582349:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203385.582362:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203385/real 1788203385] req@ffff93c1c5036680 x1875064559887360/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203385.582374:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559887360/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203385.582382:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203385.582385:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203385.582388:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559887360:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203385.582411:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559887488:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203385.582428:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203385.582438:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203385/real 1788203385] req@ffff93c1c5037800 x1875064559887488/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203385.582449:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559887488/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203385.582459:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203385.582462:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203385.582465:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559887488:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203386.286073:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203387.310016:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203388.333994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203389.358011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203390.189987:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203390.190000:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203390.190086:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203390.190107:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203390.190110:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203390.190127:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203390.382055:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203390.701908:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203390.701918:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203390.701921:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203390.701929:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2521 00000100:00080000:1.0:1788203390.701938:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203390.701943:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203390.701978:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559887616/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203390.702003:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203390.702006:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203390.702007:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203390.702010:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2521 00000100:00080000:1.0:1788203390.702012:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203390.702015:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203390.702022:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203390.702032:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559887744/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203390.702040:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203390.702042:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203390.702044:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203390.702046:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2521 00000100:00080000:1.0:1788203390.702049:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203390.702051:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203390.702055:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203390.702063:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559887616:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203390.702064:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559887872/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203390.702123:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203390.702164:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203390/real 1788203390] req@ffff93c1c5034e00 x1875064559887616/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203390.702177:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559887616/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203390.702186:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203390.702189:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203390.702194:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559887616:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203390.702199:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559887744:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203390.702213:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203390.702221:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203390/real 1788203390] req@ffff93c1c5034a80 x1875064559887744/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203390.702229:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559887744/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203390.702234:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203390.702236:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203390.702238:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559887744:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203390.702255:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559887872:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203390.702266:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203390.702272:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203390/real 1788203390] req@ffff93c1c5037100 x1875064559887872/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203390.702280:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559887872/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203390.702287:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203390.702288:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203390.702290:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559887872:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203391.405978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203392.430001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203393.454038:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203394.478085:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203395.309996:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203395.310014:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203395.310153:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203395.310180:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203395.310183:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203395.310206:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203395.501963:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203395.821951:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203395.821960:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203395.821964:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203395.821972:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2526 00000100:00080000:1.0:1788203395.821981:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203395.821987:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203395.822019:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559888000/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203395.822045:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203395.822048:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203395.822049:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203395.822052:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2526 00000100:00080000:1.0:1788203395.822054:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203395.822056:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203395.822065:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203395.822076:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559888128/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203395.822085:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203395.822088:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203395.822089:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203395.822093:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2526 00000100:00080000:1.0:1788203395.822096:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203395.822099:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203395.822102:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203395.822108:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559888256/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203395.822130:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559888000:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203395.822212:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203395.822272:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203395/real 1788203395] req@ffff93c1c5037100 x1875064559888000/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203395.822290:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559888000/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203395.822303:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203395.822306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203395.822310:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559888000:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203395.822317:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559888128:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203395.822334:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203395.822343:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203395/real 1788203395] req@ffff93c1c5034a80 x1875064559888128/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203395.822351:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559888128/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203395.822362:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203395.822364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203395.822367:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559888128:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203395.822389:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559888256:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203395.822401:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203395.822407:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203395/real 1788203395] req@ffff93c1c5034e00 x1875064559888256/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203395.822415:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559888256/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203395.822423:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203395.822426:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203395.822428:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559888256:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203396.526162:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203397.550034:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203398.574090:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203399.598044:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203400.430028:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203400.430047:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203400.430188:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203400.430211:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203400.430215:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203400.430235:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203400.622112:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203400.941996:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203400.942009:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203400.942015:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203400.942028:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2531 00000100:00080000:2.0:1788203400.942043:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203400.942052:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203400.942108:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559888384/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203400.942149:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203400.942153:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203400.942160:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203400.942166:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2531 00000100:00080000:2.0:1788203400.942169:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203400.942173:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203400.942186:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203400.942200:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559888512/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203400.942212:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203400.942218:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1788203400.942218:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559888384:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203400.942221:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203400.942232:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2531 00000100:00080000:2.0:1788203400.942235:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203400.942240:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203400.942245:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203400.942259:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559888640/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203400.942313:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203400.942375:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203400/real 1788203400] req@ffff93c0e62b8000 x1875064559888384/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203400.942397:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559888384/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203400.942409:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203400.942414:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203400.942419:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559888384:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203400.942449:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559888512:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203400.942480:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203400.942491:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203400/real 1788203400] req@ffff93c0e62b8380 x1875064559888512/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203400.942500:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559888512/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203400.942507:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203400.942509:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203400.942511:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559888512:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203400.942516:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559888640:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203400.942529:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203400.942535:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203400/real 1788203400] req@ffff93c0e62b8700 x1875064559888640/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203400.942542:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559888640/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203400.942550:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203400.942552:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203400.942554:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559888640:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203401.646001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203402.669977:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203403.694005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203404.718032:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203405.550050:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203405.550062:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203405.550148:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203405.550168:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203405.550171:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203405.550189:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203405.741998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203406.061980:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203406.061989:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203406.061993:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203406.062003:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2536 00000100:00080000:1.0:1788203406.062013:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203406.062019:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203406.062053:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559888768/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203406.062073:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203406.062076:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203406.062077:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203406.062080:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2536 00000100:00080000:1.0:1788203406.062082:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203406.062084:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203406.062093:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203406.062104:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559888896/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203406.062115:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203406.062119:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203406.062121:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203406.062127:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2536 00000100:00080000:1.0:1788203406.062131:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203406.062138:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203406.062138:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559888768:192.168.201.156@tcp:250:kworker.0 00000008:00100000:1.0:1788203406.062149:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203406.062162:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559889024/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203406.062207:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203406.062257:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203406/real 1788203406] req@ffff93c1c5037800 x1875064559888768/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203406.062279:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559888768/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203406.062291:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203406.062295:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203406.062305:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559888768:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203406.062314:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559888896:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203406.062339:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203406.062353:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203406/real 1788203406] req@ffff93c1c5036680 x1875064559888896/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203406.062366:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559888896/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203406.062376:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203406.062380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203406.062383:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559888896:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203406.062409:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559889024:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203406.062431:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203406.062450:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203406/real 1788203406] req@ffff93c1c5035c00 x1875064559889024/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203406.062463:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559889024/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203406.062480:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203406.062484:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203406.062487:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559889024:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203406.766053:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203407.790011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203408.814013:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203409.838069:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203410.670072:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203410.670090:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203410.670214:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203410.670242:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203410.670248:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203410.670283:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203410.862034:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203411.182012:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203411.182022:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203411.182028:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203411.182036:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2542 00000100:00080000:0.0:1788203411.182046:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203411.182053:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203411.182094:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559889152/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203411.182128:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203411.182132:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203411.182134:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203411.182139:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2542 00000100:00080000:0.0:1788203411.182144:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203411.182148:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203411.182158:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203411.182175:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559889280/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203411.182192:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203411.182192:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559889152:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203411.182209:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203411.182211:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203411.182218:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2542 00000100:00080000:0.0:1788203411.182224:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203411.182230:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203411.182236:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203411.182249:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559889408/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203411.182281:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203411.182324:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203411/real 1788203411] req@ffff93c0e9ce6680 x1875064559889152/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203411.182341:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559889152/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203411.182350:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203411.182355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203411.182360:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559889152:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203411.182394:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559889280:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203411.182436:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203411.182451:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203411/real 1788203411] req@ffff93c0e9ce4000 x1875064559889280/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203411.182465:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559889280/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203411.182474:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203411.182478:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203411.182481:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559889280:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203411.182490:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559889408:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203411.182511:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203411.182522:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203411/real 1788203411] req@ffff93c0e9ce6a00 x1875064559889408/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203411.182534:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559889408/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203411.182544:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203411.182548:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203411.182551:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559889408:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203411.885978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203412.909960:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203413.934006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203414.958043:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203415.790039:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203415.790054:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203415.790152:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203415.790180:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203415.790185:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203415.790214:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203415.981982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203416.302075:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203416.302087:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203416.302092:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203416.302102:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2547 00000100:00080000:2.0:1788203416.302113:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203416.302121:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203416.302164:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559889536/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203416.302194:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203416.302198:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203416.302201:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203416.302206:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2547 00000100:00080000:2.0:1788203416.302211:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203416.302217:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203416.302227:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203416.302243:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559889664/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203416.302258:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203416.302269:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1788203416.302269:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559889536:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203416.302272:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203416.302290:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2547 00000100:00080000:2.0:1788203416.302295:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203416.302300:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203416.302307:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203416.302325:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559889792/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203416.302393:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203416.302453:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203416/real 1788203416] req@ffff93c0e62bb100 x1875064559889536/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203416.302485:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559889536/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203416.302498:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203416.302503:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203416.302508:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559889536:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203416.302540:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559889664:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203416.302576:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203416.302590:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203416/real 1788203416] req@ffff93c0e62bbb80 x1875064559889664/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203416.302604:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559889664/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203416.302613:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203416.302617:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203416.302620:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559889664:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203416.302636:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559889792:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203416.302662:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203416.302674:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203416/real 1788203416] req@ffff93c0e62b9c00 x1875064559889792/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203416.302687:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559889792/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203416.302699:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203416.302703:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203416.302706:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559889792:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203417.006046:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203418.030083:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203419.054033:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203420.077958:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203420.910055:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203420.910073:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203420.910202:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203420.910225:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203420.910230:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203420.910277:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203421.102035:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203421.422013:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203421.422036:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203421.422042:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203421.422057:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2552 00000100:00080000:0.0:1788203421.422071:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203421.422081:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203421.422129:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559889920/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203421.422158:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203421.422161:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203421.422162:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203421.422166:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2552 00000100:00080000:0.0:1788203421.422169:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203421.422172:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203421.422182:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203421.422191:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559890048/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203421.422199:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203421.422203:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203421.422205:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203421.422210:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2552 00000100:00080000:0.0:1788203421.422212:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203421.422214:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203421.422224:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203421.422233:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559890176/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203421.422264:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559889920:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203421.422342:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203421.422389:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203421/real 1788203421] req@ffff93c0e9ce4380 x1875064559889920/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203421.422407:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559889920/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203421.422422:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203421.422427:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203421.422431:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559889920:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203421.422439:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559890048:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203421.422466:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203421.422479:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203421/real 1788203421] req@ffff93c0e9ce4700 x1875064559890048/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203421.422488:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559890048/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203421.422494:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203421.422497:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203421.422499:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559890048:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203421.422522:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559890176:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203421.422544:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203421.422557:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203421/real 1788203421] req@ffff93c0e9ce7480 x1875064559890176/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203421.422570:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559890176/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203421.422581:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203421.422584:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203421.422586:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559890176:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203422.126084:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203423.150059:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203424.173996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203425.198090:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203426.029997:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203426.030009:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203426.030097:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203426.030119:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203426.030123:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203426.030154:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203426.222071:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203426.541994:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203426.542006:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203426.542010:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203426.542022:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2557 00000100:00080000:2.0:1788203426.542035:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203426.542044:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203426.542091:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559890304/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203426.542124:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203426.542127:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203426.542129:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203426.542133:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2557 00000100:00080000:2.0:1788203426.542135:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203426.542138:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203426.542188:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559890304:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203426.542271:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203426.542321:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203426/real 1788203426] req@ffff93c0e62b9500 x1875064559890304/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203426.542337:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559890304/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203426.542351:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203426.542355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203426.542359:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559890304:192.168.201.156@tcp:250:kworker.0 00000008:00100000:2.0:1788203426.542379:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203426.542399:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559890432/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203426.542411:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203426.542415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203426.542418:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203426.542421:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559890432:192.168.201.156@tcp:38:kworker.0 00000100:00080000:2.0:1788203426.542427:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2557 00000100:00080000:2.0:1788203426.542434:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203426.542440:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203426.542447:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000800:00000100:3.0:1788203426.542452:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788203426.542467:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559890560/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203426.542471:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203426/real 1788203426] req@ffff93c0e62bad80 x1875064559890432/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203426.542504:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559890432/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203426.542515:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203426.542521:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203426.542526:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559890432:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203426.542544:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559890560:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203426.542573:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203426.542587:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203426/real 1788203426] req@ffff93c0e62b8a80 x1875064559890560/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203426.542601:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559890560/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203426.542612:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203426.542615:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203426.542618:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559890560:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203427.246018:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203428.269991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203429.294044:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203430.317942:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203431.150158:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203431.150175:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203431.150287:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203431.150319:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203431.150323:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203431.150348:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203431.342061:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203431.661974:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203431.661984:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203431.661989:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203431.662000:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2562 00000100:00080000:2.0:1788203431.662011:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203431.662018:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203431.662057:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559890688/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203431.662100:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203431.662104:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203431.662106:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203431.662110:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2562 00000100:00080000:2.0:1788203431.662114:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203431.662118:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203431.662128:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203431.662142:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559890816/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203431.662155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203431.662155:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559890688:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203431.662165:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203431.662167:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203431.662174:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2562 00000100:00080000:2.0:1788203431.662178:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203431.662183:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203431.662189:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203431.662203:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559890944/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203431.662216:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203431.662252:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203431/real 1788203431] req@ffff93c0e62bb800 x1875064559890688/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203431.662265:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559890688/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203431.662274:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203431.662278:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203431.662282:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559890688:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203431.662306:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559890816:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203431.662330:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203431.662340:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203431/real 1788203431] req@ffff93c0e62b9f80 x1875064559890816/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203431.662359:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559890816/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203431.662370:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203431.662372:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203431.662375:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559890816:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203431.662379:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559890944:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203431.662393:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203431.662399:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203431/real 1788203431] req@ffff93c0e62b8a80 x1875064559890944/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203431.662407:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559890944/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203431.662414:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203431.662416:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203431.662418:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559890944:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203432.365957:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203433.390094:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203434.413998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203435.438052:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203436.270065:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203436.270082:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203436.270198:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203436.270227:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203436.270232:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203436.270266:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203436.462033:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203436.781947:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203436.783329:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203436.783333:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203436.783353:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2567 00000100:00080000:2.0:1788203436.783364:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203436.783383:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203436.783436:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559891072/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203436.783468:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203436.783472:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203436.783474:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203436.783493:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2567 00000100:00080000:2.0:1788203436.783498:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203436.783503:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203436.783513:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203436.783526:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559891072:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203436.786247:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559891200/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203436.786282:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203436.786287:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203436.786290:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203436.786296:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2567 00000100:00080000:2.0:1788203436.786304:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203436.786313:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203436.786324:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203436.786346:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559891200:192.168.201.156@tcp:38:kworker.0 00000100:00100000:2.0:1788203436.786353:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559891328/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203436.786421:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559891328:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203437.486047:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203438.510032:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203439.533988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788203439.854227:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788203439.854289:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788203439.854312:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788203439.854360:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788203439.854368:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788203439.854431:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203436/real 1788203439] req@ffff93c0e62b9c00 x1875064559891328/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203439.854462:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559891328/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203439.854486:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203439.854491:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203439.854496:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559891328:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788203439.854505:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203436/real 1788203439] req@ffff93c0e62b9500 x1875064559891200/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203439.854519:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559891200/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203439.854528:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203439.854531:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203439.854535:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559891200:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203439.854542:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203436/real 1788203439] req@ffff93c0e62bad80 x1875064559891072/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203439.854554:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559891072/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203439.854562:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203439.854566:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203439.854569:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559891072:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788203440.557953:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203441.389991:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203441.390004:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203441.390093:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203441.390125:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203441.390130:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203441.390173:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203441.581997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203441.901958:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203441.901966:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203441.901969:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203441.901979:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2572 00000100:00080000:0.0:1788203441.901988:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203441.901994:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203441.902022:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559891456/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203441.902053:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203441.902057:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203441.902059:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203441.902064:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2572 00000100:00080000:0.0:1788203441.902068:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203441.902071:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203441.902081:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203441.902096:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559891584/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203441.902109:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203441.902112:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559891456:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203441.902114:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203441.902118:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203441.902126:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2572 00000100:00080000:0.0:1788203441.902129:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203441.902132:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203441.902138:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203441.902153:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559891712/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203441.902186:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203441.902238:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203441/real 1788203441] req@ffff93c0e9ce7480 x1875064559891456/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203441.902260:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559891456/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203441.902272:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203441.902276:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203441.902280:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559891456:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203441.902305:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559891584:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203441.902331:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203441.902345:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203441/real 1788203441] req@ffff93c0e9ce4700 x1875064559891584/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203441.902359:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559891584/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203441.902368:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203441.902371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203441.902374:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559891584:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203441.902381:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559891712:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203441.902405:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203441.902418:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203441/real 1788203441] req@ffff93c0e9ce4380 x1875064559891712/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203441.902431:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559891712/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203441.902450:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203441.902454:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203441.902457:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559891712:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203442.606013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203443.630080:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203444.654012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203445.678010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203446.510028:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203446.510043:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203446.510165:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203446.510195:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203446.510199:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203446.510221:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203446.702033:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203447.021917:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203447.021926:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203447.021929:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203447.021937:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2577 00000100:00080000:3.0:1788203447.021946:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203447.021952:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203447.021981:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559891840/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203447.022004:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203447.022007:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203447.022008:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203447.022011:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2577 00000100:00080000:3.0:1788203447.022013:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203447.022016:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203447.022025:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203447.022034:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559891968/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203447.022042:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203447.022046:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788203447.022047:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559891840:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203447.022048:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203447.022054:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2577 00000100:00080000:3.0:1788203447.022059:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203447.022061:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203447.022064:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203447.022073:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559892096/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203447.022130:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203447.022178:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203447/real 1788203447] req@ffff93c1c4a4e680 x1875064559891840/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203447.022200:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559891840/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203447.022213:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203447.022218:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203447.022222:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559891840:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203447.022247:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559891968:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203447.022274:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203447.022289:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203447/real 1788203447] req@ffff93c1c4a4dc00 x1875064559891968/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203447.022303:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559891968/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203447.022312:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203447.022315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203447.022319:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559891968:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203447.022328:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559892096:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203447.022351:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203447.022364:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203447/real 1788203447] req@ffff93c1c4a4fb80 x1875064559892096/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203447.022376:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559892096/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203447.022386:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203447.022390:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203447.022405:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559892096:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203447.726065:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203448.750026:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203449.774057:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203450.798035:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203451.630081:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203451.630102:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203451.630244:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203451.630277:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203451.630281:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203451.630310:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203451.822063:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203452.141915:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203452.141923:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203452.141926:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203452.141934:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2583 00000100:00080000:3.0:1788203452.141944:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203452.141950:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203452.141984:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559892224/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203452.142008:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203452.142011:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203452.142012:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203452.142016:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2583 00000100:00080000:3.0:1788203452.142018:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203452.142020:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203452.142028:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203452.142036:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559892352/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203452.142044:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203452.142047:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203452.142048:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203452.142051:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2583 00000100:00080000:3.0:1788203452.142053:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203452.142055:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203452.142058:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203452.142065:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559892224:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203452.142068:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559892480/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203452.142123:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203452.142155:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203452/real 1788203452] req@ffff93c1c4a4c000 x1875064559892224/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203452.142169:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559892224/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203452.142176:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203452.142179:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203452.142182:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559892224:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203452.142187:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559892352:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203452.142202:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203452.142210:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203452/real 1788203452] req@ffff93c1c4a4ea00 x1875064559892352/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203452.142217:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559892352/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203452.142223:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203452.142224:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203452.142226:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559892352:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203452.142244:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559892480:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203452.142255:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203452.142261:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203452/real 1788203452] req@ffff93c1c4a4f480 x1875064559892480/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203452.142268:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559892480/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203452.142275:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203452.142277:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203452.142279:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559892480:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203452.846048:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203453.870080:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203454.894006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203455.917922:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203456.749965:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203456.749982:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203456.750092:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203456.750114:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203456.750118:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203456.750139:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788203456.941936:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203457.261947:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203457.261955:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203457.261965:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203457.261975:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2588 00000100:00080000:1.0:1788203457.261988:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203457.261996:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203457.262036:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559892608/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203457.262069:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203457.262073:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203457.262075:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203457.262080:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2588 00000100:00080000:1.0:1788203457.262086:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203457.262091:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203457.262103:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559892608:192.168.201.156@tcp:250:kworker.0 00000008:00100000:1.0:1788203457.262104:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203457.262124:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559892736/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203457.262138:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203457.262142:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203457.262144:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203457.262148:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2588 00000100:00080000:1.0:1788203457.262153:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203457.262157:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000800:00000100:3.0:1788203457.262162:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000008:00100000:1.0:1788203457.262163:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203457.262180:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559892864/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203457.262207:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203457/real 1788203457] req@ffff93c1c5035c00 x1875064559892608/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203457.262228:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559892608/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203457.262239:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203457.262243:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203457.262247:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559892608:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203457.262271:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559892736:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203457.262296:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203457.262311:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203457/real 1788203457] req@ffff93c1c5036680 x1875064559892736/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203457.262325:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559892736/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203457.262337:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203457.262339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203457.262342:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559892736:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203457.262346:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559892864:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203457.262363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203457.262371:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203457/real 1788203457] req@ffff93c1c5037800 x1875064559892864/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203457.262378:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559892864/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203457.262385:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203457.262388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203457.262390:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559892864:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203457.966060:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203458.990017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203460.014076:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203461.038076:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203461.870052:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203461.870065:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203461.870188:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203461.870218:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203461.870223:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203461.870256:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203462.062058:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203462.381975:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203462.381985:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203462.381988:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203462.381997:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2593 00000100:00080000:3.0:1788203462.382007:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203462.382013:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203462.382046:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559892992/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203462.382075:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203462.382078:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203462.382079:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203462.382082:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2593 00000100:00080000:3.0:1788203462.382084:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203462.382087:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203462.382094:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203462.382103:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559893120/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203462.382110:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203462.382117:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203462.382119:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788203462.382124:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559892992:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203462.382126:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2593 00000100:00080000:3.0:1788203462.382132:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203462.382136:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203462.382140:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203462.382149:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559893248/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203462.382188:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203462.382231:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203462/real 1788203462] req@ffff93c1c4a4c380 x1875064559892992/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203462.382246:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559892992/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203462.382255:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203462.382258:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203462.382262:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559892992:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203462.382269:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559893120:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203462.382284:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203462.382292:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203462/real 1788203462] req@ffff93c1c4a4c700 x1875064559893120/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203462.382299:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559893120/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203462.382304:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203462.382306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203462.382308:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559893120:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203462.382326:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559893248:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203462.382337:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203462.382344:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203462/real 1788203462] req@ffff93c1c4a4e300 x1875064559893248/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203462.382356:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559893248/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203462.382372:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203462.382376:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203462.382379:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559893248:192.168.201.156@tcp:8:kworker.0 00010000:00010000:1.0:1788203463.086022:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203464.110101:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203465.134069:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203466.158056:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203466.990011:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203466.990030:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203466.990160:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203466.990193:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203466.990197:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203466.990227:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788203467.182028:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203467.501965:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203467.501979:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203467.501984:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203467.501995:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2598 00000100:00080000:2.0:1788203467.502017:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203467.502025:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203467.502073:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559893376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203467.502099:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203467.502102:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203467.502103:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203467.502107:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2598 00000100:00080000:2.0:1788203467.502109:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203467.502112:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203467.502122:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203467.502134:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559893504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203467.502143:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203467.502147:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203467.502149:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788203467.502151:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559893376:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203467.502155:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2598 00000100:00080000:2.0:1788203467.502161:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203467.502164:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203467.502167:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203467.502176:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559893632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203467.502230:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203467.502274:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203467/real 1788203467] req@ffff93c0e62bbb80 x1875064559893376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203467.502296:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559893376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203467.502309:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203467.502314:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203467.502319:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559893376:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203467.502349:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559893504:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203467.502369:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203467.502379:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203467/real 1788203467] req@ffff93c0e62bb100 x1875064559893504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203467.502388:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559893504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203467.502394:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203467.502396:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203467.502399:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559893504:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203467.502404:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559893632:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203467.502417:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203467.502423:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203467/real 1788203467] req@ffff93c0e62b8700 x1875064559893632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203467.502431:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559893632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203467.502452:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203467.502455:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203467.502458:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559893632:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203468.206040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203469.230003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203470.254057:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203471.277996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203472.110049:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203472.110071:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203472.110204:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203472.110240:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203472.110246:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203472.110280:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203472.302088:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203472.622000:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203472.622014:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203472.622019:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203472.622033:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2603 00000100:00080000:1.0:1788203472.622048:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203472.622058:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203472.622120:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559893760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203472.622152:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203472.622156:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203472.622158:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203472.622163:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2603 00000100:00080000:1.0:1788203472.622166:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203472.622171:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203472.622183:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203472.622204:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559893888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203472.622215:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203472.622220:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203472.622223:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203472.622229:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2603 00000100:00080000:1.0:1788203472.622234:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203472.622238:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203472.622244:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203472.622254:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559894016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203472.622279:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559893760:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788203472.622376:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203472.622417:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203472/real 1788203472] req@ffff93c1c5034e00 x1875064559893760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203472.622439:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559893760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203472.622451:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203472.622456:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203472.622460:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559893760:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203472.622469:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559893888:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203472.622491:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203472.622500:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203472/real 1788203472] req@ffff93c1c5034a80 x1875064559893888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203472.622510:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559893888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203472.622515:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203472.622518:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203472.622520:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559893888:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203472.622540:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559894016:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203472.622552:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203472.622558:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203472/real 1788203472] req@ffff93c1c5037100 x1875064559894016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203472.622566:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559894016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203472.622574:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203472.622575:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203472.622577:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559894016:192.168.201.156@tcp:8:kworker.0 00010000:00010000:1.0:1788203473.326035:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203474.350004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203475.374053:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203476.398036:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203477.230072:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203477.230090:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203477.230196:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203477.230220:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203477.230223:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203477.230240:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788203477.422086:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203477.741975:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203477.741986:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203477.741990:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203477.742001:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2608 00000100:00080000:3.0:1788203477.742015:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203477.742022:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203477.742071:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559894144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203477.742102:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203477.742105:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203477.742106:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203477.742110:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2608 00000100:00080000:3.0:1788203477.742112:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203477.742116:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203477.742127:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203477.742136:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559894272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203477.742143:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203477.742146:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203477.742147:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203477.742150:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2608 00000100:00080000:3.0:1788203477.742152:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203477.742154:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203477.742157:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203477.742163:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559894400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203477.742205:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559894144:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788203477.742312:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203477.742356:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203477/real 1788203477] req@ffff93c1c4a4d180 x1875064559894144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203477.742373:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559894144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203477.742387:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203477.742390:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203477.742394:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559894144:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203477.742399:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559894272:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203477.742416:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203477.742424:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203477/real 1788203477] req@ffff93c1c4a4f800 x1875064559894272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203477.742438:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559894272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203477.742446:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203477.742449:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203477.742453:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559894272:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203477.742459:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559894400:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203477.742483:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203477.742495:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203477/real 1788203477] req@ffff93c1c4a4d880 x1875064559894400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203477.742505:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559894400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203477.742513:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203477.742515:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203477.742517:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559894400:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203478.445970:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203479.470060:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203480.494099:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203481.518030:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203482.349988:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203482.350005:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203482.350097:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203482.350118:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203482.350121:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203482.350138:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203482.542014:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203482.861971:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203482.861980:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203482.861984:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203482.861992:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2613 00000100:00080000:1.0:1788203482.862002:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203482.862008:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203482.862044:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559894528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203482.862068:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203482.862071:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203482.862073:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203482.862076:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2613 00000100:00080000:1.0:1788203482.862078:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203482.862081:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203482.862088:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203482.862098:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559894656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203482.862106:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203482.862108:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203482.862109:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203482.862112:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2613 00000100:00080000:1.0:1788203482.862115:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203482.862117:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203482.862120:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203482.862123:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559894528:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203482.862130:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559894784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203482.862201:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203482.862236:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203482/real 1788203482] req@ffff93c1c5037100 x1875064559894528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203482.862249:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559894528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203482.862256:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203482.862265:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203482.862268:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559894528:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203482.862274:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559894656:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203482.862291:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203482.862300:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203482/real 1788203482] req@ffff93c1c5034a80 x1875064559894656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203482.862307:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559894656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203482.862313:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203482.862315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203482.862317:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559894656:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203482.862335:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559894784:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203482.862346:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203482.862352:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203482/real 1788203482] req@ffff93c1c5034e00 x1875064559894784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203482.862360:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559894784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203482.862368:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203482.862369:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203482.862371:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559894784:192.168.201.156@tcp:8:kworker.0 00010000:00010000:1.0:1788203483.566022:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203484.589994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203485.614035:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203486.637991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203487.469997:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203487.470009:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203487.470134:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203487.470156:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203487.470159:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203487.470181:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788203487.662007:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203487.982045:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203487.982054:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203487.982057:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203487.982066:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2618 00000100:00080000:3.0:1788203487.982076:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203487.982081:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203487.982146:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d500 x1875064559894912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203487.982171:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203487.982173:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203487.982175:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203487.982178:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2618 00000100:00080000:3.0:1788203487.982181:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203487.982183:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203487.982190:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203487.982200:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ca80 x1875064559895040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203487.982208:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203487.982210:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203487.982211:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203487.982214:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2618 00000100:00080000:3.0:1788203487.982217:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788203487.982221:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559894912:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203487.982223:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203487.982233:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203487.982248:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559895168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203487.982280:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203487.982315:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203487/real 1788203487] req@ffff93c1c4a4d500 x1875064559894912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203487.982328:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d500 x1875064559894912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203487.982335:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203487.982338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203487.982341:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559894912:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203487.982346:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559895040:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203487.982360:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203487.982368:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203487/real 1788203487] req@ffff93c1c4a4ca80 x1875064559895040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203487.982375:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ca80 x1875064559895040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203487.982385:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203487.982387:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203487.982390:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559895040:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203487.982407:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559895168:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203487.982418:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203487.982424:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203487/real 1788203487] req@ffff93c1c4a4d880 x1875064559895168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203487.982431:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559895168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203487.982438:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203487.982440:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203487.982442:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559895168:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203488.686017:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203489.709981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203490.733983:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203491.758067:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203492.590000:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203492.590037:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203492.590140:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203492.590165:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203492.590168:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203492.590200:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203492.781928:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203493.102021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203493.102029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203493.102033:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203493.102042:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2623 00000100:00080000:2.0:1788203493.102051:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203493.102057:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203493.102092:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559895296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203493.102117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203493.102120:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203493.102122:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203493.102124:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2623 00000100:00080000:2.0:1788203493.102127:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203493.102130:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203493.102137:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203493.102146:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559895424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203493.102153:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203493.102156:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203493.102157:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203493.102160:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2623 00000100:00080000:2.0:1788203493.102162:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203493.102164:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203493.102168:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203493.102172:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559895296:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203493.102178:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559895552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203493.102239:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203493.102279:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203493/real 1788203493] req@ffff93c0e62b8380 x1875064559895296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203493.102296:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559895296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203493.102307:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203493.102311:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203493.102315:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559895296:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203493.102323:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559895424:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203493.102358:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203493.102373:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203493/real 1788203493] req@ffff93c0e62b8000 x1875064559895424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203493.102385:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559895424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203493.102394:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203493.102398:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203493.102401:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559895424:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203493.102425:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559895552:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203493.102446:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203493.102457:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203493/real 1788203493] req@ffff93c0e62ba680 x1875064559895552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203493.102469:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559895552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203493.102479:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203493.102482:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203493.102485:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559895552:192.168.201.156@tcp:8:kworker.0 00010000:00010000:1.0:1788203493.806048:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203494.830068:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203495.853994:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203496.878074:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203497.710083:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203497.710099:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203497.710202:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203497.710224:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203497.710227:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203497.710245:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788203497.902065:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203498.221994:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203498.222002:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203498.222006:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203498.222014:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2629 00000100:00080000:2.0:1788203498.222024:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203498.222030:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203498.222065:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559895680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203498.222089:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203498.222092:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203498.222094:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203498.222096:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2629 00000100:00080000:2.0:1788203498.222099:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203498.222101:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203498.222108:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203498.222116:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559895808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203498.222123:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203498.222126:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203498.222127:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203498.222130:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2629 00000100:00080000:2.0:1788203498.222131:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203498.222134:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203498.222137:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203498.222147:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559895936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203498.222204:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559895680:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788203498.222306:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203498.222376:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203498/real 1788203498] req@ffff93c0e62bb480 x1875064559895680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203498.222402:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559895680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203498.222415:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203498.222420:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203498.222425:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559895680:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203498.222435:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559895808:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203498.222461:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203498.222477:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203498/real 1788203498] req@ffff93c0e62baa00 x1875064559895808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203498.222490:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559895808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203498.222500:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203498.222504:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203498.222508:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559895808:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203498.222516:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559895936:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203498.222538:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203498.222551:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203498/real 1788203498] req@ffff93c0e62ba680 x1875064559895936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203498.222564:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559895936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203498.222583:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203498.222586:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203498.222590:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559895936:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203498.926064:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203499.950124:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203500.974047:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203501.998057:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203502.830065:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203502.830086:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203502.830250:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203502.830293:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203502.830300:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203502.830338:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203503.022935:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203503.341937:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203503.341945:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203503.341949:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203503.341958:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2634 00000100:00080000:2.0:1788203503.341968:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203503.341974:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203503.342009:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559896064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203503.342038:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203503.342042:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203503.342044:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203503.342049:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2634 00000100:00080000:2.0:1788203503.342052:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203503.342057:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203503.342067:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203503.342079:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559896192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203503.342088:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203503.342091:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203503.342092:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203503.342095:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2634 00000100:00080000:2.0:1788203503.342098:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788203503.342107:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559896064:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203503.342109:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203503.342123:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203503.342146:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559896320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203503.342244:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559896192:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203503.342272:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559896320:192.168.201.156@tcp:8:kworker.0 00010000:00010000:1.0:1788203504.046052:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203505.070020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203506.094009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788203506.414143:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788203506.414188:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788203506.414207:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788203506.414255:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788203506.414266:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788203506.414324:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203503/real 1788203506] req@ffff93c0e62b8380 x1875064559896192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203506.414355:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559896192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203506.414369:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203506.414375:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203506.414379:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559896192:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203506.414396:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203503/real 1788203506] req@ffff93c0e62b8700 x1875064559896320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203506.414411:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559896320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203506.414424:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203506.414428:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203506.414432:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559896320:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788203506.414441:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203503/real 1788203506] req@ffff93c0e62b8000 x1875064559896064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203506.414454:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559896064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203506.414462:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203506.414466:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203506.414470:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559896064:192.168.201.156@tcp:250:kworker.0 00010000:00010000:2.0:1788203507.118083:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203507.950042:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203507.950064:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203507.950198:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203507.950233:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203507.950238:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203507.950270:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788203508.142047:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203508.461979:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203508.461987:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203508.461990:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203508.462000:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2639 00000100:00080000:1.0:1788203508.462011:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203508.462018:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203508.462054:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559896448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203508.462092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203508.462096:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203508.462098:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203508.462101:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2639 00000100:00080000:1.0:1788203508.462105:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203508.462108:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203508.462115:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203508.462129:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559896576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203508.462139:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203508.462143:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203508.462144:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203508.462149:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2639 00000100:00080000:1.0:1788203508.462152:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203508.462155:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203508.462160:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203508.462172:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559896704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203508.462276:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559896448:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788203508.462357:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203508.462410:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203508/real 1788203508] req@ffff93c1c5037800 x1875064559896448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203508.462432:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559896448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203508.462444:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203508.462449:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203508.462454:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559896448:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203508.462462:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559896576:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203508.462488:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203508.462501:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203508/real 1788203508] req@ffff93c1c5036680 x1875064559896576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203508.462519:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559896576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203508.462528:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203508.462531:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203508.462535:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559896576:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203508.462541:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559896704:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203508.462564:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203508.462575:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203508/real 1788203508] req@ffff93c1c5035c00 x1875064559896704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203508.462586:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559896704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203508.462596:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203508.462600:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203508.462604:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559896704:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203509.166023:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203510.189963:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203511.214069:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203512.238061:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203513.070052:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203513.070067:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203513.070179:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203513.070209:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203513.070213:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203513.070246:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203513.262058:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203513.581943:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203513.581952:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203513.581957:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203513.581967:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2644 00000100:00080000:1.0:1788203513.581978:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203513.581985:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203513.582025:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559896832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203513.582054:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203513.582057:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203513.582058:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203513.582061:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2644 00000100:00080000:1.0:1788203513.582064:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203513.582066:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203513.582074:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203513.582082:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559896960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203513.582089:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203513.582092:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203513.582093:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203513.582096:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2644 00000100:00080000:1.0:1788203513.582098:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203513.582100:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203513.582103:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203513.582110:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559896832:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203513.582112:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559897088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203513.582171:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203513.582211:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203513/real 1788203513] req@ffff93c1c5035c00 x1875064559896832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203513.582225:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559896832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203513.582239:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203513.582242:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203513.582246:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559896832:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203513.582252:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559896960:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203513.582267:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203513.582275:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203513/real 1788203513] req@ffff93c1c5036680 x1875064559896960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203513.582284:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559896960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203513.582290:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203513.582292:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203513.582294:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559896960:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203513.582313:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559897088:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203513.582325:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203513.582331:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203513/real 1788203513] req@ffff93c1c5037800 x1875064559897088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203513.582338:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559897088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203513.582345:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203513.582347:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203513.582349:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559897088:192.168.201.156@tcp:8:kworker.0 00010000:00010000:1.0:1788203514.286055:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203515.309998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203516.334054:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203517.358002:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203518.190009:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203518.190023:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203518.190136:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203518.190160:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203518.190163:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203518.190186:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788203518.381991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203518.701987:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203518.701997:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203518.702001:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203518.702011:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2649 00000100:00080000:1.0:1788203518.702021:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203518.702028:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203518.702064:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559897216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203518.702093:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203518.702097:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203518.702099:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203518.702103:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2649 00000100:00080000:1.0:1788203518.702106:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203518.702110:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203518.702120:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203518.702133:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559897344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203518.702145:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788203518.702147:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559897216:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788203518.702150:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203518.702154:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203518.702160:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2649 00000100:00080000:1.0:1788203518.702163:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203518.702167:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203518.702173:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203518.702185:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559897472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203518.702220:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203518.702271:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203518/real 1788203518] req@ffff93c1c5034e00 x1875064559897216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203518.702294:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559897216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203518.702306:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203518.702311:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203518.702315:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559897216:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203518.702339:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559897344:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203518.702363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203518.702383:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203518/real 1788203518] req@ffff93c1c5034a80 x1875064559897344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203518.702398:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559897344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203518.702407:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203518.702411:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203518.702415:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559897344:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203518.702423:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559897472:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203518.702442:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203518.702452:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203518/real 1788203518] req@ffff93c1c5037100 x1875064559897472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203518.702466:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559897472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203518.702476:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203518.702479:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203518.702482:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559897472:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203519.405972:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203520.430029:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203521.454010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203522.478305:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203523.309988:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203523.310003:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203523.310112:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203523.310136:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203523.310140:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203523.310159:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203523.502029:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203523.821984:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203523.821993:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203523.821996:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203523.822005:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2654 00000100:00080000:3.0:1788203523.822013:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203523.822019:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203523.822051:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559897600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203523.822074:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203523.822078:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203523.822080:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203523.822084:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2654 00000100:00080000:3.0:1788203523.822088:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203523.822092:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203523.822103:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203523.822117:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559897728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203523.822126:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203523.822130:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203523.822131:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788203523.822132:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559897600:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203523.822135:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2654 00000100:00080000:3.0:1788203523.822138:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203523.822141:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203523.822146:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203523.822155:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559897856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203523.822191:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203523.822247:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203523/real 1788203523] req@ffff93c1c4a4f800 x1875064559897600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203523.822269:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559897600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203523.822280:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203523.822283:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203523.822286:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559897600:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203523.822316:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559897728:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203523.822342:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203523.822355:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203523/real 1788203523] req@ffff93c1c4a4d180 x1875064559897728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203523.822364:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559897728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203523.822376:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203523.822380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203523.822384:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559897728:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203523.822390:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559897856:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203523.822412:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203523.822423:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203523/real 1788203523] req@ffff93c1c4a4e300 x1875064559897856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203523.822436:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559897856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203523.822443:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203523.822445:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203523.822447:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559897856:192.168.201.156@tcp:8:kworker.0 00010000:00010000:1.0:1788203524.526043:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203525.550007:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203526.574031:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203527.597968:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203528.430009:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203528.430024:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203528.430113:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203528.430134:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203528.430137:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203528.430155:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788203528.622177:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203528.941971:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203528.941982:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203528.941985:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203528.941994:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2659 00000100:00080000:3.0:1788203528.942005:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203528.942023:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203528.942063:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559897984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203528.942089:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203528.942092:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203528.942094:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203528.942097:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2659 00000100:00080000:3.0:1788203528.942100:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203528.942102:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203528.942110:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203528.942120:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559898112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203528.942127:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203528.942130:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203528.942132:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203528.942135:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2659 00000100:00080000:3.0:1788203528.942137:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203528.942142:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203528.942144:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559897984:192.168.201.156@tcp:250:kworker.0 00000008:00100000:3.0:1788203528.942145:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203528.942158:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559898240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203528.942209:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203528.942253:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203528/real 1788203528] req@ffff93c1c4a4c700 x1875064559897984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203528.942269:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559897984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203528.942284:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203528.942288:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203528.942291:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559897984:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203528.942297:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559898112:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203528.942313:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203528.942323:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203528/real 1788203528] req@ffff93c1c4a4c380 x1875064559898112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203528.942331:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559898112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203528.942337:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203528.942339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203528.942341:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559898112:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203528.942360:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559898240:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203528.942394:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203528.942404:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203528/real 1788203528] req@ffff93c1c4a4f480 x1875064559898240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203528.942412:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559898240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203528.942420:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203528.942422:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203528.942424:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559898240:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203529.645993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203530.670009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203531.694048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203532.717996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203533.550002:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203533.550016:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203533.550105:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203533.550126:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203533.550129:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203533.550147:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203533.742027:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203534.061907:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203534.061915:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203534.061919:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203534.061927:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2664 00000100:00080000:3.0:1788203534.061937:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203534.061943:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203534.061977:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559898368/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203534.062000:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203534.062002:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203534.062004:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203534.062007:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2664 00000100:00080000:3.0:1788203534.062009:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203534.062012:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203534.062019:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203534.062027:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559898496/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203534.062036:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203534.062038:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203534.062039:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203534.062044:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2664 00000100:00100000:0.0:1788203534.062046:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559898368:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203534.062046:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203534.062052:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203534.062057:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203534.062066:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559898624/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203534.062106:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203534.062145:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203534/real 1788203534] req@ffff93c1c4a4ea00 x1875064559898368/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203534.062159:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559898368/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203534.062167:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203534.062170:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203534.062174:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559898368:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203534.062193:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559898496:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203534.062215:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203534.062224:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203534/real 1788203534] req@ffff93c1c4a4c000 x1875064559898496/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203534.062232:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559898496/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203534.062238:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203534.062240:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203534.062242:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559898496:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203534.062246:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559898624:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203534.062258:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203534.062264:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203534/real 1788203534] req@ffff93c1c4a4fb80 x1875064559898624/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203534.062302:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559898624/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203534.062319:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203534.062322:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203534.062324:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559898624:192.168.201.156@tcp:8:kworker.0 00010000:00010000:1.0:1788203534.765989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203535.790026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203536.814008:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203537.838054:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203538.670052:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203538.670070:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203538.670177:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203538.670218:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203538.670223:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203538.670245:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788203538.862043:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203539.181987:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203539.181997:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203539.182001:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203539.182012:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2670 00000100:00080000:3.0:1788203539.182023:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203539.182031:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203539.182091:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559898752/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203539.182121:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203539.182125:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203539.182127:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203539.182131:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2670 00000100:00080000:3.0:1788203539.182134:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203539.182137:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203539.182147:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203539.182161:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559898880/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203539.182179:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203539.182185:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788203539.182188:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559898752:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203539.182191:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203539.182224:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2670 00000100:00080000:3.0:1788203539.182231:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203539.182239:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203539.182247:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203539.182270:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559899008/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203539.182307:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203539.182360:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203539/real 1788203539] req@ffff93c1c4a4dc00 x1875064559898752/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203539.182387:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559898752/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203539.182403:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203539.182409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203539.182414:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559898752:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203539.182447:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559898880:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203539.182481:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203539.182497:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203539/real 1788203539] req@ffff93c1c4a4e680 x1875064559898880/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203539.182509:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559898880/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203539.182518:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203539.182521:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203539.182524:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559898880:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203539.182531:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559899008:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203539.182549:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203539.182564:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203539/real 1788203539] req@ffff93c1c4a4ed80 x1875064559899008/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203539.182600:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559899008/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203539.182615:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203539.182620:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203539.182625:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559899008:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203539.886078:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788203540.910043:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203541.934067:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203542.958009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203543.790014:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203543.790027:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203543.790148:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203543.790178:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203543.790183:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203543.790209:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203543.982042:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203544.301917:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203544.301925:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203544.301929:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203544.301937:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2675 00000100:00080000:3.0:1788203544.301947:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203544.301960:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203544.301998:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f100 x1875064559899136/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203544.302020:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203544.302023:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203544.302024:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203544.302027:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2675 00000100:00080000:3.0:1788203544.302030:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203544.302033:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203544.302041:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203544.302050:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4df80 x1875064559899264/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203544.302058:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203544.302060:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203544.302061:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203544.302064:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2675 00000100:00080000:3.0:1788203544.302066:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788203544.302072:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559899136:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203544.302072:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203544.302078:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203544.302089:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559899392/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203544.302159:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203544.302207:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203544/real 1788203544] req@ffff93c1c4a4f100 x1875064559899136/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203544.302228:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f100 x1875064559899136/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203544.302240:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203544.302244:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203544.302249:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559899136:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203544.302256:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559899264:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203544.302277:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203544.302289:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203544/real 1788203544] req@ffff93c1c4a4df80 x1875064559899264/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203544.302301:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4df80 x1875064559899264/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203544.302310:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203544.302314:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203544.302317:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559899264:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203544.302342:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559899392:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203544.302361:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203544.302370:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203544/real 1788203544] req@ffff93c1c4a4ed80 x1875064559899392/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203544.302379:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559899392/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203544.302386:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203544.302388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203544.302390:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559899392:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203545.005933:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203546.030019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203547.053932:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203548.078001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203548.910066:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203548.910082:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203548.910192:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203548.910214:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203548.910218:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203548.910244:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203549.101946:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203549.421995:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203549.422003:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203549.422014:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203549.422037:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2680 00000100:00080000:2.0:1788203549.422047:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203549.422054:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203549.422088:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559899520/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203549.422116:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203549.422119:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203549.422120:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203549.422123:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2680 00000100:00080000:2.0:1788203549.422125:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203549.422132:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203549.422140:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203549.422152:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559899648/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203549.422168:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203549.422171:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203549.422172:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203549.422175:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2680 00000100:00080000:2.0:1788203549.422178:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788203549.422189:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559899520:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203549.422189:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203549.422205:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203549.422223:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559899776/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203549.422295:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203549.422341:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203549/real 1788203549] req@ffff93c0e62bb100 x1875064559899520/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203549.422362:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559899520/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203549.422374:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203549.422378:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203549.422383:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559899520:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203549.422426:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559899648:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203549.422447:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203549.422458:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203549/real 1788203549] req@ffff93c0e62bbb80 x1875064559899648/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203549.422466:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559899648/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203549.422473:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203549.422475:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203549.422477:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559899648:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203549.422482:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559899776:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203549.422494:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203549.422500:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203549/real 1788203549] req@ffff93c0e62bad80 x1875064559899776/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203549.422508:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559899776/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203549.422520:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203549.422523:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203549.422525:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559899776:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203550.126107:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203551.149940:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203552.174045:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203553.198028:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203554.030005:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203554.030024:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203554.030124:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203554.030151:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203554.030154:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203554.030174:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203554.222038:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203554.541979:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203554.541988:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203554.541991:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203554.542000:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2685 00000100:00080000:2.0:1788203554.542009:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203554.542016:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203554.542051:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559899904/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203554.542076:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203554.542079:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203554.542080:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203554.542084:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2685 00000100:00080000:2.0:1788203554.542086:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203554.542088:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203554.542097:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203554.542105:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559900032/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203554.542113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203554.542115:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203554.542117:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203554.542120:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2685 00000100:00080000:2.0:1788203554.542122:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203554.542124:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203554.542128:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203554.542133:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559899904:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203554.542139:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559900160/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203554.542197:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203554.542239:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203554/real 1788203554] req@ffff93c0e62b9500 x1875064559899904/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203554.542256:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559899904/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203554.542268:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203554.542273:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203554.542277:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559899904:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203554.542285:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559900032:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203554.542308:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203554.542320:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203554/real 1788203554] req@ffff93c0e62b9c00 x1875064559900032/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203554.542329:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559900032/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203554.542334:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203554.542337:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203554.542339:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559900032:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203554.542358:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559900160:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203554.542371:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203554.542377:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203554/real 1788203554] req@ffff93c0e62b8a80 x1875064559900160/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203554.542384:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559900160/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203554.542391:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203554.542393:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203554.542394:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559900160:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203555.246031:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203556.270034:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203557.294017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203558.318074:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203559.150049:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203559.150063:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203559.150175:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203559.150205:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203559.150210:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203559.150241:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203559.341991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203559.661966:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203559.661976:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203559.661980:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203559.661991:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2690 00000100:00080000:2.0:1788203559.662000:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203559.662006:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203559.662047:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559900288/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203559.662071:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203559.662074:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203559.662075:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203559.662078:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2690 00000100:00080000:2.0:1788203559.662080:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203559.662082:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203559.662089:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203559.662098:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559900416/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203559.662105:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203559.662108:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203559.662109:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203559.662115:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2690 00000100:00080000:2.0:1788203559.662118:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203559.662120:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203559.662123:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203559.662130:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559900544/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203559.662149:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559900288:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788203559.662229:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203559.662274:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203559/real 1788203559] req@ffff93c0e62b9f80 x1875064559900288/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203559.662296:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559900288/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203559.662310:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203559.662315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203559.662321:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559900288:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203559.662331:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559900416:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203559.662352:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203559.662362:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203559/real 1788203559] req@ffff93c0e62bb800 x1875064559900416/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203559.662371:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559900416/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203559.662378:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203559.662380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203559.662383:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559900416:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203559.662408:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559900544:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203559.662432:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203559.662445:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203559/real 1788203559] req@ffff93c0e62b8a80 x1875064559900544/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203559.662459:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559900544/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203559.662469:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203559.662471:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203559.662474:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559900544:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203560.365989:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203561.389978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203562.414068:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203563.437987:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203564.270048:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203564.270061:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203564.270167:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203564.270190:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203564.270194:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203564.270213:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203564.462039:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203564.781988:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203564.782000:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203564.782004:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203564.782015:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2695 00000100:00080000:3.0:1788203564.782028:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203564.782036:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203564.782078:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559900672/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203564.782103:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203564.782105:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203564.782110:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203564.782114:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2695 00000100:00080000:3.0:1788203564.782117:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203564.782120:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203564.782128:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203564.782138:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559900800/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203564.782146:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203564.782149:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203564.782150:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203564.782156:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2695 00000100:00100000:0.0:1788203564.782157:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559900672:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203564.782158:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203564.782164:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203564.782173:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203564.782182:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559900928/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203564.782235:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203564.782282:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203564/real 1788203564] req@ffff93c1c4a4e680 x1875064559900672/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203564.782304:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559900672/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203564.782315:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203564.782319:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203564.782323:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559900672:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203564.782345:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559900800:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203564.782365:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203564.782376:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203564/real 1788203564] req@ffff93c1c4a4dc00 x1875064559900800/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203564.782388:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559900800/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203564.782397:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203564.782400:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203564.782403:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559900800:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203564.782409:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559900928:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203564.782424:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203564.782433:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203564/real 1788203564] req@ffff93c1c4a4fb80 x1875064559900928/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203564.782451:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559900928/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203564.782461:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203564.782463:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203564.782466:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559900928:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203565.485992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203566.510014:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203567.533980:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203568.557957:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203569.389984:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203569.390000:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203569.390093:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203569.390114:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203569.390118:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203569.390135:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203569.582023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203569.902025:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203569.902035:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203569.902039:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203569.902049:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2700 00000100:00080000:1.0:1788203569.902060:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203569.902066:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203569.902106:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559901056/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203569.902138:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203569.902143:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203569.902145:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203569.902149:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2700 00000100:00080000:1.0:1788203569.902153:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203569.902158:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203569.902168:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203569.902183:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559901184/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203569.902195:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559901056:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788203569.902198:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203569.902206:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203569.902208:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203569.902214:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2700 00000100:00080000:1.0:1788203569.902218:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203569.902222:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203569.902227:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203569.902241:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559901312/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203569.902323:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559901184:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203569.902374:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559901312:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203570.606005:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203571.629971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203572.654033:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788203572.974184:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:0.0:1788203572.974243:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788203572.974262:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788203572.974302:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:0.0:1788203572.974314:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788203572.974333:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203569/real 1788203572] req@ffff93c1c5034a80 x1875064559901184/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203572.974364:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559901184/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203572.974376:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203572.974381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203572.974385:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559901184:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203572.974394:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203569/real 1788203572] req@ffff93c1c5034e00 x1875064559901312/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203572.974409:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559901312/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203572.974421:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203572.974424:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203572.974426:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559901312:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788203572.974432:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203569/real 1788203572] req@ffff93c1c5037100 x1875064559901056/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203572.974442:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559901056/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203572.974450:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203572.974454:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203572.974458:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559901056:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788203573.678016:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203574.510097:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203574.510114:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203574.510241:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203574.510276:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203574.510282:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203574.510321:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203574.702029:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203575.021917:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203575.021926:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203575.021929:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203575.021937:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2705 00000100:00080000:0.0:1788203575.021946:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203575.021952:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203575.021986:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559901440/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203575.022014:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203575.022017:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203575.022019:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203575.022022:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2705 00000100:00080000:0.0:1788203575.022025:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203575.022028:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203575.022036:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203575.022048:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559901568/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203575.022057:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203575.022057:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559901440:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203575.022063:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203575.022066:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203575.022071:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2705 00000100:00080000:0.0:1788203575.022073:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203575.022076:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203575.022080:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203575.022090:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559901696/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203575.022135:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203575.022191:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203575/real 1788203575] req@ffff93c0e9ce6a00 x1875064559901440/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203575.022212:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559901440/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203575.022224:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203575.022229:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203575.022232:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559901440:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203575.022256:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559901568:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203575.022284:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203575.022299:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203575/real 1788203575] req@ffff93c0e9ce4000 x1875064559901568/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203575.022312:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559901568/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203575.022321:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203575.022324:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203575.022328:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559901568:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203575.022335:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559901696:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203575.022359:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203575.022372:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203575/real 1788203575] req@ffff93c0e9ce6680 x1875064559901696/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203575.022393:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559901696/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203575.022404:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203575.022408:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203575.022410:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559901696:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203575.725946:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203576.750092:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203577.774021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203578.798028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203579.630007:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203579.630020:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203579.630129:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203579.630150:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203579.630153:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203579.630171:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203579.821977:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203580.141962:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203580.141971:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203580.141975:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203580.141983:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2711 00000100:00080000:0.0:1788203580.141993:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203580.141998:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203580.142036:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559901824/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203580.142067:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203580.142070:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203580.142071:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203580.142075:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2711 00000100:00080000:0.0:1788203580.142078:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203580.142081:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203580.142090:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203580.142100:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559901952/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203580.142108:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203580.142110:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203580.142112:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203580.142114:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2711 00000100:00080000:0.0:1788203580.142117:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203580.142121:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203580.142123:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559901824:192.168.201.156@tcp:250:kworker.0 00000008:00100000:0.0:1788203580.142125:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203580.142136:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559902080/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203580.142183:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203580.142226:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203580/real 1788203580] req@ffff93c0e9ce7b80 x1875064559901824/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203580.142240:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559901824/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203580.142249:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203580.142252:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203580.142256:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559901824:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203580.142264:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559901952:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203580.142279:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203580.142288:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203580/real 1788203580] req@ffff93c0e9ce5c00 x1875064559901952/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203580.142295:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559901952/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203580.142305:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203580.142308:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203580.142310:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559901952:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203580.142329:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559902080:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203580.142340:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203580.142346:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203580/real 1788203580] req@ffff93c0e9ce6d80 x1875064559902080/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203580.142353:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559902080/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203580.142360:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203580.142361:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203580.142363:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559902080:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203580.846063:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203581.869930:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203582.893961:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203583.917949:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203584.750023:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203584.750042:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203584.750156:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203584.750179:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203584.750182:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203584.750199:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203584.942039:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203585.261990:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203585.262005:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203585.262011:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203585.262026:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2716 00000100:00080000:1.0:1788203585.262042:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203585.262052:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203585.262118:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559902208/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203585.262154:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203585.262158:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203585.262160:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203585.262165:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2716 00000100:00080000:1.0:1788203585.262169:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203585.262173:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203585.262185:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203585.262203:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559902336/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203585.262217:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203585.262221:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203585.262225:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203585.262230:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559902208:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788203585.262233:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2716 00000100:00080000:1.0:1788203585.262240:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203585.262246:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203585.262253:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203585.262266:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559902464/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203585.262346:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203585.262431:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203585/real 1788203585] req@ffff93c1c5037800 x1875064559902208/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203585.262455:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559902208/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203585.262467:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203585.262471:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203585.262476:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559902208:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203585.262506:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559902336:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203585.262531:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203585.262545:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203585/real 1788203585] req@ffff93c1c5036680 x1875064559902336/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203585.262557:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559902336/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203585.262566:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203585.262570:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203585.262573:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559902336:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203585.262580:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559902464:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203585.262599:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203585.262610:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203585/real 1788203585] req@ffff93c1c5035c00 x1875064559902464/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203585.262621:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559902464/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203585.262632:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203585.262636:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203585.262640:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559902464:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203585.965971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203586.990019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203588.014014:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203589.038040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203589.870022:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203589.870035:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203589.870133:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203589.870153:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203589.870155:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203589.870174:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203590.061966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203590.381963:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203590.381975:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203590.381979:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203590.381990:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2721 00000100:00080000:2.0:1788203590.382000:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203590.382008:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203590.382059:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559902592/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203590.382084:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203590.382088:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203590.382090:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203590.382095:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2721 00000100:00080000:2.0:1788203590.382099:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203590.382103:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203590.382115:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203590.382133:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559902720/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203590.382140:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559902592:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203590.382165:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203590.382168:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203590.382170:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203590.382174:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2721 00000100:00080000:2.0:1788203590.382178:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203590.382181:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203590.382186:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203590.382198:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559902848/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203590.382211:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203590.382260:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203590/real 1788203590] req@ffff93c0e62b9c00 x1875064559902592/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203590.382281:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559902592/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203590.382293:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203590.382298:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203590.382301:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559902592:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203590.382342:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559902720:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203590.382365:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203590.382377:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203590/real 1788203590] req@ffff93c0e62b9500 x1875064559902720/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203590.382391:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559902720/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203590.382400:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203590.382404:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203590.382408:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559902720:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203590.382414:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559902848:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203590.382430:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203590.382440:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203590/real 1788203590] req@ffff93c0e62bad80 x1875064559902848/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203590.382452:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559902848/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203590.382468:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203590.382472:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203590.382475:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559902848:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203591.086087:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203592.109989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203593.134022:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203594.158008:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203594.990401:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203594.990414:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203594.990505:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203594.990526:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203594.990530:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203594.990550:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203595.181967:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203595.501972:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203595.501992:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203595.501997:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203595.502009:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2726 00000100:00080000:2.0:1788203595.502022:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203595.502031:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203595.502078:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559902976/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203595.502112:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203595.502115:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203595.502117:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203595.502122:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2726 00000100:00080000:2.0:1788203595.502125:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203595.502129:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203595.502140:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203595.502154:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559903104/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203595.502164:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203595.502169:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1788203595.502169:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559902976:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203595.502174:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203595.502181:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2726 00000100:00080000:2.0:1788203595.502184:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203595.502187:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203595.502192:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203595.502204:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559903232/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203595.502251:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203595.502299:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203595/real 1788203595] req@ffff93c0e62bbb80 x1875064559902976/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203595.502319:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559902976/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203595.502330:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203595.502334:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203595.502338:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559902976:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203595.502370:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559903104:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203595.502392:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203595.502405:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203595/real 1788203595] req@ffff93c0e62bb100 x1875064559903104/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203595.502416:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559903104/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203595.502424:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203595.502427:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203595.502430:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559903104:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203595.502437:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559903232:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203595.502452:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203595.502460:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203595/real 1788203595] req@ffff93c0e62b8000 x1875064559903232/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203595.502467:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559903232/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203595.502474:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203595.502476:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203595.502478:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559903232:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203596.205951:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203597.230099:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203598.253980:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203599.278004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203600.110047:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203600.110066:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203600.110204:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203600.110247:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203600.110252:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203600.110283:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203600.301963:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203600.621950:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203600.621965:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203600.621970:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203600.621983:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2731 00000100:00080000:2.0:1788203600.621997:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203600.622006:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203600.622054:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559903360/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203600.622087:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203600.622091:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203600.622093:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203600.622099:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2731 00000100:00080000:2.0:1788203600.622103:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203600.622108:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203600.622120:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203600.622148:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559903488/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203600.622160:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203600.622167:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559903360:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203600.622168:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203600.622173:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203600.622182:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2731 00000100:00080000:2.0:1788203600.622186:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203600.622191:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203600.622197:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203600.622212:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559903616/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203600.622229:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203600.622279:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203600/real 1788203600] req@ffff93c0e62b8700 x1875064559903360/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203600.622296:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559903360/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203600.622304:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203600.622307:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203600.622310:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559903360:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203600.622329:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559903488:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203600.622345:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203600.622352:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203600/real 1788203600] req@ffff93c0e62b8380 x1875064559903488/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203600.622364:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559903488/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203600.622371:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203600.622374:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203600.622378:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559903488:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203600.622384:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559903616:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203600.622413:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203600.622425:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203600/real 1788203600] req@ffff93c0e62ba680 x1875064559903616/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203600.622448:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559903616/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203600.622457:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203600.622461:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203600.622464:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559903616:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203601.326041:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203602.349981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203603.373988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203604.398026:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203605.230072:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203605.230092:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203605.230229:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203605.230254:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203605.230258:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203605.230280:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203605.421976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203605.741946:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203605.741957:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203605.741962:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203605.741972:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2736 00000100:00080000:2.0:1788203605.741984:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203605.741990:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203605.742053:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559903744/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203605.742083:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203605.742086:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203605.742088:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203605.742091:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2736 00000100:00080000:2.0:1788203605.742094:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203605.742097:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203605.742106:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203605.742118:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559903872/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203605.742126:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203605.742129:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203605.742130:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203605.742133:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2736 00000100:00080000:2.0:1788203605.742135:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203605.742137:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203605.742142:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203605.742146:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559903744:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203605.742153:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559904000/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203605.742212:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203605.742259:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203605/real 1788203605] req@ffff93c0e62baa00 x1875064559903744/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203605.742276:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559903744/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203605.742285:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203605.742289:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203605.742292:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559903744:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203605.742298:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559903872:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203605.742313:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203605.742322:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203605/real 1788203605] req@ffff93c0e62bb480 x1875064559903872/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203605.742330:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559903872/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203605.742336:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203605.742338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203605.742340:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559903872:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203605.742372:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559904000:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203605.742386:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203605.742393:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203605/real 1788203605] req@ffff93c0e62ba680 x1875064559904000/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203605.742402:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559904000/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203605.742414:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203605.742416:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203605.742418:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559904000:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203606.445968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203607.470030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203608.493960:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203609.518036:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203610.350013:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203610.350028:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203610.350143:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203610.350179:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203610.350185:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203610.350226:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203610.541978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203610.861999:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203610.862010:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203610.862014:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203610.862025:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2741 00000100:00080000:3.0:1788203610.862036:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203610.862043:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203610.862079:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559904128/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203610.862117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203610.862121:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203610.862123:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203610.862127:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2741 00000100:00080000:3.0:1788203610.862131:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203610.862136:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203610.862146:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203610.862158:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559904128:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203610.862167:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559904256/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203610.862180:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203610.862184:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203610.862187:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203610.862192:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2741 00000100:00080000:3.0:1788203610.862196:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203610.862201:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203610.862206:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000800:00000100:0.0:1788203610.862215:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203610.862230:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559904384/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203610.862265:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203610/real 1788203610] req@ffff93c1c4a4c000 x1875064559904128/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203610.862282:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559904128/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203610.862291:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203610.862294:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203610.862297:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559904128:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203610.862323:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559904256:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203610.862339:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203610.862348:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203610/real 1788203610] req@ffff93c1c4a4ea00 x1875064559904256/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203610.862357:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559904256/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203610.862364:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203610.862368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203610.862372:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559904256:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203610.862378:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559904384:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203610.862399:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203610.862411:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203610/real 1788203610] req@ffff93c1c4a4f480 x1875064559904384/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203610.862424:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559904384/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203610.862431:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203610.862433:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203610.862435:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559904384:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203611.566031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203612.589971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203613.614033:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203614.637958:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203615.470093:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203615.470106:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203615.470203:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203615.470223:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203615.470226:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203615.470244:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203615.662033:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203615.981957:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203615.981968:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203615.981972:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203615.981987:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2746 00000100:00080000:3.0:1788203615.981999:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203615.982007:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203615.982040:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559904512/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203615.982067:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203615.982070:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203615.982074:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203615.982078:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2746 00000100:00080000:3.0:1788203615.982081:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203615.982084:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203615.982091:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203615.982100:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559904640/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203615.982109:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203615.982112:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788203615.982115:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559904512:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203615.982115:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203615.982123:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2746 00000100:00080000:3.0:1788203615.982126:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203615.982128:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203615.982131:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203615.982141:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559904768/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203615.982178:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203615.982219:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203615/real 1788203615] req@ffff93c1c4a4c380 x1875064559904512/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203615.982241:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559904512/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203615.982252:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203615.982257:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203615.982261:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559904512:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203615.982284:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559904640:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203615.982303:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203615.982312:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203615/real 1788203615] req@ffff93c1c4a4c700 x1875064559904640/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203615.982321:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559904640/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203615.982327:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203615.982329:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203615.982331:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559904640:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203615.982335:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559904768:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203615.982347:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203615.982353:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203615/real 1788203615] req@ffff93c1c4a4e300 x1875064559904768/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203615.982360:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559904768/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203615.982366:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203615.982368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203615.982370:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559904768:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203616.685944:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203617.710026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203618.734009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203619.758030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203620.590002:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203620.590025:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203620.590140:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203620.590171:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203620.590175:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203620.590199:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203620.782024:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203621.101948:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203621.101960:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203621.101965:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203621.101976:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2751 00000100:00080000:3.0:1788203621.101990:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203621.101999:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203621.102051:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559904896/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203621.102077:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203621.102080:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203621.102082:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203621.102085:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2751 00000100:00080000:3.0:1788203621.102088:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203621.102091:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203621.102101:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203621.102110:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559905024/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203621.102120:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203621.102123:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203621.102125:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203621.102128:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2751 00000100:00080000:3.0:1788203621.102130:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203621.102132:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203621.102137:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203621.102145:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559905152/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203621.102158:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559904896:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788203621.102226:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203621.102263:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203621/real 1788203621] req@ffff93c1c4a4d180 x1875064559904896/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203621.102280:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559904896/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203621.102292:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203621.102296:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203621.102300:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559904896:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203621.102309:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559905024:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203621.102335:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203621.102349:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203621/real 1788203621] req@ffff93c1c4a4f800 x1875064559905024/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203621.102362:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559905024/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203621.102371:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203621.102375:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203621.102378:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559905024:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203621.102404:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559905152:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203621.102420:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203621.102429:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203621/real 1788203621] req@ffff93c1c4a4d880 x1875064559905152/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203621.102441:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559905152/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203621.102450:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203621.102453:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203621.102456:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559905152:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203621.806066:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203622.830025:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203623.854009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203624.877919:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203625.710026:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203625.710039:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203625.710127:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203625.710147:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203625.710149:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203625.710166:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203625.902052:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203626.221961:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203626.221969:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203626.221972:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203626.221981:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2757 00000100:00080000:3.0:1788203626.221990:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203626.221996:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203626.222031:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ca80 x1875064559905280/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203626.222057:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203626.222060:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203626.222061:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203626.222064:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2757 00000100:00080000:3.0:1788203626.222066:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203626.222069:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203626.222078:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203626.222091:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d500 x1875064559905408/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203626.222102:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203626.222106:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203626.222109:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203626.222115:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2757 00000100:00100000:0.0:1788203626.222125:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559905280:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203626.222128:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203626.222137:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203626.222143:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203626.222160:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559905536/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203626.222196:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203626.222250:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203626/real 1788203626] req@ffff93c1c4a4ca80 x1875064559905280/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203626.222272:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ca80 x1875064559905280/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203626.222285:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203626.222290:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203626.222294:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559905280:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203626.222302:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559905408:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203626.222326:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203626.222339:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203626/real 1788203626] req@ffff93c1c4a4d500 x1875064559905408/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203626.222353:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d500 x1875064559905408/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203626.222363:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203626.222367:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203626.222371:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559905408:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203626.222395:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559905536:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203626.222411:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203626.222418:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203626/real 1788203626] req@ffff93c1c4a4d880 x1875064559905536/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203626.222426:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559905536/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203626.222434:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203626.222436:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203626.222438:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559905536:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203626.926025:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203627.950052:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203628.974013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203629.998015:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203630.829980:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203630.829994:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203630.830090:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203630.830112:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203630.830114:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203630.830132:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203631.021916:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203631.341949:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203631.341958:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203631.341961:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203631.341974:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2762 00000100:00080000:3.0:1788203631.341984:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203631.341989:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203631.342023:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559905664/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203631.342046:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203631.342049:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203631.342050:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203631.342053:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2762 00000100:00080000:3.0:1788203631.342056:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203631.342058:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203631.342066:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203631.342075:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559905792/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203631.342083:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203631.342086:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203631.342087:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203631.342090:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2762 00000100:00080000:3.0:1788203631.342093:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788203631.342096:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559905664:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203631.342098:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203631.342105:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203631.342113:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559905920/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203631.342169:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203631.342203:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203631/real 1788203631] req@ffff93c1c4a4f800 x1875064559905664/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203631.342219:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559905664/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203631.342244:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203631.342248:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203631.342253:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559905664:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203631.342261:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559905792:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203631.342282:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203631.342293:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203631/real 1788203631] req@ffff93c1c4a4d180 x1875064559905792/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203631.342306:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559905792/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203631.342315:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203631.342318:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203631.342322:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559905792:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203631.342346:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559905920:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203631.342363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203631.342371:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203631/real 1788203631] req@ffff93c1c4a4e300 x1875064559905920/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203631.342379:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559905920/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203631.342386:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203631.342389:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203631.342391:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559905920:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203632.046018:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203633.069948:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203634.094045:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203635.117988:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203635.950069:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203635.950092:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203635.950232:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203635.950266:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203635.950271:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203635.950306:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203636.141998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203636.461972:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203636.461979:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203636.461982:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203636.461989:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2767 00000100:00080000:2.0:1788203636.461998:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203636.462004:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203636.462035:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559906048/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203636.462055:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203636.462057:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203636.462059:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203636.462061:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2767 00000100:00080000:2.0:1788203636.462063:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203636.462065:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203636.462073:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203636.462085:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559906176/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203636.462094:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203636.462097:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203636.462099:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203636.462104:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2767 00000100:00100000:0.0:1788203636.462109:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559906048:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203636.462110:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203636.462117:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203636.462122:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203636.462131:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559906304/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203636.462205:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559906176:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203636.462236:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559906304:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203637.165985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203638.190019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203639.214002:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788203639.534093:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788203639.534151:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788203639.534171:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788203639.534225:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788203639.534236:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788203639.534314:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203636/real 1788203639] req@ffff93c0e62b8000 x1875064559906304/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203639.534345:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559906304/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203639.534362:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203639.534368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203639.534373:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559906304:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788203639.534381:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203636/real 1788203639] req@ffff93c0e62b8380 x1875064559906048/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203639.534394:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559906048/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203639.534402:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203639.534405:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203639.534409:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559906048:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203639.534419:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203636/real 1788203639] req@ffff93c0e62b8700 x1875064559906176/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203639.534430:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559906176/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203639.534437:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203639.534441:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203639.534444:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559906176:192.168.201.156@tcp:38:kworker.0 00010000:00010000:2.0:1788203640.238038:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203641.069977:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203641.069992:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203641.070109:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203641.070132:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203641.070136:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203641.070155:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203641.262022:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203641.581970:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203641.581980:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203641.581984:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203641.581994:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2772 00000100:00080000:3.0:1788203641.582004:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203641.582012:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203641.582044:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559906432/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203641.582070:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203641.582074:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203641.582076:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203641.582086:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2772 00000100:00080000:3.0:1788203641.582090:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203641.582095:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203641.582104:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203641.582116:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559906432:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203641.582120:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559906560/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203641.582142:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203641.582146:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203641.582147:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203641.582153:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2772 00000100:00080000:3.0:1788203641.582157:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203641.582160:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203641.582166:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203641.582178:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559906688/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203641.582232:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203641.582334:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203641/real 1788203641] req@ffff93c1c4a4c700 x1875064559906432/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203641.582365:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559906432/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203641.582377:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203641.582382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203641.582387:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559906432:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203641.582459:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559906560:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203641.582486:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203641.582500:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203641/real 1788203641] req@ffff93c1c4a4c380 x1875064559906560/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203641.582518:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559906560/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203641.582526:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203641.582530:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203641.582533:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559906560:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203641.582547:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559906688:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203641.582568:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203641.582580:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203641/real 1788203641] req@ffff93c1c4a4f480 x1875064559906688/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203641.582591:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559906688/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203641.582602:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203641.582606:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203641.582609:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559906688:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203642.286003:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203643.310006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203644.334060:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203645.357946:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203646.190034:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203646.190050:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203646.190157:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203646.190181:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203646.190184:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203646.190203:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203646.381965:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203646.701978:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203646.701988:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203646.701992:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203646.702001:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2777 00000100:00080000:2.0:1788203646.702010:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203646.702016:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203646.702045:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559906816/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203646.702066:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203646.702068:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203646.702070:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203646.702072:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2777 00000100:00080000:2.0:1788203646.702074:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203646.702076:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203646.702083:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203646.702090:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559906944/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203646.702101:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203646.702104:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203646.702104:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203646.702107:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2777 00000100:00080000:2.0:1788203646.702108:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203646.702110:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203646.702114:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203646.702116:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559906816:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203646.702122:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559907072/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203646.702180:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203646.702215:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203646/real 1788203646] req@ffff93c0e62bb100 x1875064559906816/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203646.702228:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559906816/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203646.702235:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203646.702237:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203646.702240:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559906816:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203646.702244:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559906944:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203646.702257:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203646.702263:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203646/real 1788203646] req@ffff93c0e62bbb80 x1875064559906944/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203646.702270:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559906944/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203646.702275:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203646.702276:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203646.702278:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559906944:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203646.702294:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559907072:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203646.702304:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203646.702309:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203646/real 1788203646] req@ffff93c0e62bad80 x1875064559907072/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203646.702315:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559907072/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203646.702321:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203646.702323:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203646.702324:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559907072:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203647.405983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203648.430021:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203649.453971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203650.478022:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203651.310005:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203651.310019:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203651.310123:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203651.310151:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203651.310155:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203651.310184:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203651.501949:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203651.821938:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203651.821946:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203651.821950:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203651.821957:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2782 00000100:00080000:2.0:1788203651.821966:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203651.821977:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203651.822008:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559907200/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203651.822029:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203651.822031:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203651.822032:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203651.822035:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2782 00000100:00080000:2.0:1788203651.822037:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203651.822039:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203651.822046:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203651.822054:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559907328/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203651.822061:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203651.822063:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203651.822064:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203651.822066:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2782 00000100:00080000:2.0:1788203651.822068:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788203651.822072:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559907200:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203651.822073:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203651.822079:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203651.822087:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559907456/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203651.822131:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203651.822180:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203651/real 1788203651] req@ffff93c0e62b9500 x1875064559907200/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203651.822195:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559907200/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203651.822205:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203651.822209:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203651.822212:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559907200:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203651.822220:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559907328:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203651.822250:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203651.822261:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203651/real 1788203651] req@ffff93c0e62b9c00 x1875064559907328/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203651.822269:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559907328/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203651.822274:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203651.822276:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203651.822278:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559907328:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203651.822296:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559907456:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203651.822309:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203651.822315:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203651/real 1788203651] req@ffff93c0e62b8a80 x1875064559907456/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203651.822322:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559907456/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203651.822327:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203651.822329:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203651.822330:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559907456:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203652.526048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203653.549968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203654.574064:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203655.597983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203656.430038:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203656.430061:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203656.430221:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203656.430256:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203656.430262:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203656.430300:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203656.622083:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203656.941925:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203656.941937:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203656.941942:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203656.941953:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2787 00000100:00080000:2.0:1788203656.941966:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203656.941973:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203656.942019:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559907584/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203656.942047:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203656.942049:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203656.942051:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203656.942055:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2787 00000100:00080000:2.0:1788203656.942057:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203656.942060:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203656.942070:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203656.942080:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559907712/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203656.942089:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788203656.942090:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559907584:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203656.942093:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203656.942097:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203656.942103:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2787 00000100:00080000:2.0:1788203656.942106:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203656.942108:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203656.942112:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203656.942123:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559907840/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203656.942192:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203656.942243:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203656/real 1788203656] req@ffff93c0e62bb800 x1875064559907584/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203656.942266:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559907584/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203656.942279:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203656.942284:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203656.942288:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559907584:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203656.942316:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559907712:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203656.942343:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203656.942358:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203656/real 1788203656] req@ffff93c0e62b9f80 x1875064559907712/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203656.942379:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559907712/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203656.942386:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203656.942388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203656.942391:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559907712:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203656.942396:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559907840:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203656.942413:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203656.942421:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203656/real 1788203656] req@ffff93c0e62b8a80 x1875064559907840/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203656.942429:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559907840/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203656.942437:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203656.942438:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203656.942440:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559907840:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203657.645990:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203658.670062:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203659.693973:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203660.718079:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203661.550087:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203661.550100:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203661.550229:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203661.550269:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203661.550274:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203661.550297:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203661.741967:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203662.061962:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203662.061970:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203662.061974:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203662.061983:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2792 00000100:00080000:3.0:1788203662.061992:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203662.061998:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203662.062031:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559907968/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203662.062052:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203662.062055:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203662.062057:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203662.062070:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2792 00000100:00080000:3.0:1788203662.062074:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203662.062078:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203662.062087:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203662.062097:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559908096/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203662.062105:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203662.062108:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203662.062109:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203662.062112:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2792 00000100:00080000:3.0:1788203662.062115:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203662.062120:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203662.062122:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559907968:192.168.201.156@tcp:250:kworker.0 00000008:00100000:3.0:1788203662.062125:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203662.062134:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559908224/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203662.062201:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203662.062246:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203662/real 1788203662] req@ffff93c1c4a4ea00 x1875064559907968/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203662.062274:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559907968/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203662.062286:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203662.062291:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203662.062295:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559907968:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203662.062304:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559908096:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203662.062331:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203662.062345:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203662/real 1788203662] req@ffff93c1c4a4c000 x1875064559908096/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203662.062354:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559908096/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203662.062360:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203662.062362:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203662.062365:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559908096:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203662.062383:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559908224:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203662.062399:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203662.062406:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203662/real 1788203662] req@ffff93c1c4a4fb80 x1875064559908224/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203662.062415:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559908224/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203662.062432:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203662.062436:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203662.062440:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559908224:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203662.766019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203663.789910:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203664.814063:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203665.838034:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203666.670068:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203666.670094:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203666.670223:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203666.670253:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203666.670258:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203666.670286:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203666.862038:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203667.181954:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203667.181968:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203667.181985:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203667.181998:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2798 00000100:00080000:2.0:1788203667.182013:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203667.182022:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203667.182074:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559908352/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203667.182104:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203667.182109:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203667.182110:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203667.182115:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2798 00000100:00080000:2.0:1788203667.182120:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203667.182125:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203667.182138:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203667.182155:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559908480/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203667.182166:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559908352:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203667.182168:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203667.182187:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203667.182191:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203667.182197:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2798 00000100:00080000:2.0:1788203667.182202:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203667.182208:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203667.182216:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203667.182231:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559908608/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203667.182256:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203667.182304:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203667/real 1788203667] req@ffff93c0e62b9c00 x1875064559908352/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203667.182322:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559908352/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203667.182332:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203667.182336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203667.182340:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559908352:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203667.182363:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559908480:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203667.182387:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203667.182397:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203667/real 1788203667] req@ffff93c0e62b9500 x1875064559908480/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203667.182405:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559908480/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203667.182410:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203667.182412:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203667.182414:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559908480:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203667.182419:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559908608:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203667.182431:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203667.182436:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203667/real 1788203667] req@ffff93c0e62bad80 x1875064559908608/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203667.182443:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559908608/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203667.182449:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203667.182451:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203667.182453:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559908608:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203667.886010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203668.909988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203669.934043:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203670.958011:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203671.790033:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203671.790049:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203671.790155:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203671.790176:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203671.790179:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203671.790195:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203671.981985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203672.302028:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203672.302039:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203672.302042:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203672.302053:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2803 00000100:00080000:0.0:1788203672.302064:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203672.302070:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203672.302110:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559908736/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203672.302136:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203672.302139:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203672.302140:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203672.302143:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2803 00000100:00080000:0.0:1788203672.302145:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203672.302148:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203672.302157:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203672.302165:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559908864/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203672.302172:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203672.302175:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203672.302176:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203672.302179:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2803 00000100:00080000:0.0:1788203672.302180:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203672.302182:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203672.302185:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203672.302192:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559908736:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203672.302194:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559908992/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203672.302266:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203672.302318:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203672/real 1788203672] req@ffff93c0e9ce5f80 x1875064559908736/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203672.302335:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559908736/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203672.302344:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203672.302349:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203672.302353:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559908736:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203672.302361:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559908864:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203672.302383:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203672.302396:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203672/real 1788203672] req@ffff93c0e9ce4a80 x1875064559908864/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203672.302408:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559908864/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203672.302416:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203672.302419:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203672.302423:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559908864:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203672.302451:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559908992:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203672.302469:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203672.302477:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203672/real 1788203672] req@ffff93c0e9ce7100 x1875064559908992/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203672.302488:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559908992/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203672.302498:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203672.302501:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203672.302503:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559908992:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203673.005971:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203674.029936:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203675.054012:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203676.077913:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203676.909994:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203676.910007:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203676.910093:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203676.910111:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203676.910114:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203676.910129:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203677.102037:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203677.421975:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203677.421982:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203677.421985:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203677.421993:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2808 00000100:00080000:2.0:1788203677.422001:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203677.422006:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203677.422040:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559909120/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203677.422062:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203677.422064:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203677.422066:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203677.422069:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2808 00000100:00080000:2.0:1788203677.422071:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203677.422073:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203677.422080:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203677.422087:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559909248/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203677.422093:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203677.422095:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203677.422096:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203677.422099:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2808 00000100:00080000:2.0:1788203677.422101:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203677.422104:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203677.422107:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203677.422113:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559909376/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203677.422124:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559909120:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203677.422190:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203677.422219:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203677/real 1788203677] req@ffff93c0e62bbb80 x1875064559909120/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203677.422232:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559909120/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203677.422242:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203677.422245:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203677.422249:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559909120:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203677.422256:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559909248:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203677.422277:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203677.422287:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203677/real 1788203677] req@ffff93c0e62bb100 x1875064559909248/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203677.422294:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559909248/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203677.422299:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203677.422301:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203677.422303:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559909248:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203677.422320:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559909376:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203677.422331:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203677.422337:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203677/real 1788203677] req@ffff93c0e62b8700 x1875064559909376/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203677.422348:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559909376/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203677.422354:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203677.422356:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203677.422358:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559909376:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203678.125981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203679.150010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203680.174005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203681.198002:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203682.029965:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203682.029982:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203682.030116:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203682.030147:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203682.030151:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203682.030184:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203682.222015:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203682.541936:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203682.541945:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203682.541949:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203682.541960:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2813 00000100:00080000:2.0:1788203682.541971:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203682.541978:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203682.542023:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559909504/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203682.542050:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203682.542053:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203682.542054:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203682.542058:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2813 00000100:00080000:2.0:1788203682.542060:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203682.542063:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203682.542073:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203682.542082:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559909632/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203682.542089:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203682.542092:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203682.542093:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203682.542097:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2813 00000100:00080000:2.0:1788203682.542100:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788203682.542103:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559909504:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203682.542104:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203682.542109:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203682.542116:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559909760/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203682.542179:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203682.542220:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203682/real 1788203682] req@ffff93c0e62b8380 x1875064559909504/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203682.542235:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559909504/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203682.542243:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203682.542246:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203682.542249:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559909504:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203682.542254:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559909632:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203682.542268:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203682.542275:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203682/real 1788203682] req@ffff93c0e62b8000 x1875064559909632/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203682.542282:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559909632/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203682.542292:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203682.542294:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203682.542296:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559909632:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203682.542316:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559909760:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203682.542326:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203682.542331:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203682/real 1788203682] req@ffff93c0e62ba680 x1875064559909760/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203682.542338:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559909760/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203682.542345:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203682.542346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203682.542348:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559909760:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203683.245952:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203684.270008:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203685.294013:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203686.317999:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203687.150333:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203687.150347:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203687.150456:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203687.150482:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203687.150486:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203687.150515:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203687.342021:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203687.661899:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203687.661912:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203687.661916:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203687.661923:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2818 00000100:00080000:2.0:1788203687.661932:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203687.661937:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203687.661965:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559909888/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203687.661993:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203687.661997:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203687.661999:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203687.662003:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2818 00000100:00080000:2.0:1788203687.662007:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203687.662010:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203687.662024:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203687.662037:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559909888:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203687.662042:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559910016/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203687.662055:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203687.662059:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203687.662061:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203687.662065:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2818 00000100:00080000:2.0:1788203687.662069:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203687.662072:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203687.662077:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203687.662087:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559910144/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203687.662103:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203687.662145:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203687/real 1788203687] req@ffff93c0e62bb480 x1875064559909888/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203687.662162:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559909888/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203687.662171:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203687.662174:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203687.662177:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559909888:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203687.662199:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559910016:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203687.662221:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203687.662232:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203687/real 1788203687] req@ffff93c0e62baa00 x1875064559910016/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203687.662243:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559910016/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203687.662250:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203687.662253:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203687.662256:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559910016:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203687.662261:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559910144:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203687.662284:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203687.662293:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203687/real 1788203687] req@ffff93c0e62ba680 x1875064559910144/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203687.662300:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559910144/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203687.662307:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203687.662309:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203687.662311:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559910144:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203688.365952:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203689.390045:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203690.413970:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203691.438050:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203692.270046:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203692.270065:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203692.270185:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203692.270214:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203692.270219:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203692.270246:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203692.462023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203692.782706:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203692.782718:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203692.782723:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203692.782733:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2823 00000100:00080000:1.0:1788203692.782746:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203692.782757:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203692.782810:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559910272/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203692.782880:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203692.782885:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559910272:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788203692.782886:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203692.782891:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203692.782896:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2823 00000100:00080000:1.0:1788203692.782900:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203692.782904:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203692.782915:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203692.782930:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559910400/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203692.782941:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203692.782945:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203692.782947:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203692.782951:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2823 00000100:00080000:1.0:1788203692.782955:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788203692.782958:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00080000:1.0:1788203692.782962:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203692.782968:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203692.782980:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559910528/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203692.783021:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203692/real 1788203692] req@ffff93c1c5035c00 x1875064559910272/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203692.783039:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559910272/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203692.783051:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203692.783054:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203692.783057:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559910272:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203692.783079:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559910400:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203692.783098:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203692.783110:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203692/real 1788203692] req@ffff93c1c5036680 x1875064559910400/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203692.783124:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559910400/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203692.783132:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203692.783135:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203692.783138:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559910400:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203692.783142:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559910528:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203692.783157:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203692.783165:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203692/real 1788203692] req@ffff93c1c5037800 x1875064559910528/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203692.783172:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559910528/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203692.783184:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203692.783186:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203692.783188:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559910528:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203693.486742:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203694.510003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203695.534030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203696.557998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203697.390020:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203697.390042:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203697.390172:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203697.390211:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203697.390217:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203697.390248:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203697.582023:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203697.901942:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203697.901953:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203697.901956:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203697.901966:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2828 00000100:00080000:1.0:1788203697.901978:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203697.901985:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203697.902033:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559910656/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203697.902055:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203697.902058:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203697.902060:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203697.902063:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2828 00000100:00080000:1.0:1788203697.902065:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203697.902067:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203697.902076:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203697.902085:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559910784/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203697.902092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203697.902094:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203697.902096:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203697.902099:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2828 00000100:00080000:1.0:1788203697.902100:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203697.902102:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203697.902107:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203697.902116:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559910912/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203697.902116:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559910656:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203697.902195:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203697.902244:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203697/real 1788203697] req@ffff93c1c5037100 x1875064559910656/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203697.902266:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559910656/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203697.902278:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203697.902282:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203697.902286:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559910656:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203697.902294:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559910784:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203697.902313:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203697.902323:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203697/real 1788203697] req@ffff93c1c5034e00 x1875064559910784/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203697.902335:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559910784/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203697.902343:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203697.902346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203697.902349:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559910784:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203697.902373:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559910912:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203697.902392:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203697.902404:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203697/real 1788203697] req@ffff93c1c5034a80 x1875064559910912/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203697.902415:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559910912/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203697.902425:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203697.902428:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203697.902432:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559910912:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203698.605950:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203699.630048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203700.654008:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203701.677991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203702.510036:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203702.510056:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203702.510180:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203702.510205:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203702.510208:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203702.510227:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203702.701908:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203703.021903:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203703.021917:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203703.021922:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203703.021945:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2833 00000100:00080000:1.0:1788203703.021960:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203703.021969:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203703.022025:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559911040/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203703.022052:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203703.022055:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203703.022056:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203703.022059:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2833 00000100:00080000:1.0:1788203703.022062:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203703.022065:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203703.022075:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203703.022086:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559911168/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203703.022093:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203703.022096:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203703.022097:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203703.022100:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2833 00000100:00080000:1.0:1788203703.022101:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203703.022104:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203703.022107:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203703.022116:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559911296/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203703.022119:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559911040:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203703.022244:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559911168:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203703.022288:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559911296:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203703.726036:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203704.750005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203705.774008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788203706.094077:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788203706.094116:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788203706.094133:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788203706.094171:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788203706.094180:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:3.0:1788203706.094228:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203703/real 1788203706] req@ffff93c1c5037100 x1875064559911296/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203706.094253:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559911296/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203706.094267:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203706.094272:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203706.094276:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559911296:192.168.201.156@tcp:8:kworker.0 00000100:00100000:3.0:1788203706.094283:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203703/real 1788203706] req@ffff93c1c5034a80 x1875064559911040/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203706.094305:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559911040/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203706.094313:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203706.094317:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203706.094320:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559911040:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203706.094327:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203703/real 1788203706] req@ffff93c1c5034e00 x1875064559911168/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203706.094339:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559911168/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203706.094355:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203706.094359:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203706.094363:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559911168:192.168.201.156@tcp:38:kworker.0 00010000:00010000:0.0:1788203706.797995:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203707.630168:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203707.630180:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203707.630257:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203707.630286:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203707.630289:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203707.630314:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203707.822032:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203708.141983:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203708.141993:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203708.141998:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203708.142007:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2839 00000100:00080000:2.0:1788203708.142017:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203708.142024:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203708.142068:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559911424/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203708.142092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203708.142096:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203708.142098:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203708.142102:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2839 00000100:00080000:2.0:1788203708.142106:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203708.142110:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203708.142123:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203708.142146:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559911552/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203708.142148:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559911424:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203708.142165:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203708.142168:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203708.142170:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203708.142175:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2839 00000100:00080000:2.0:1788203708.142179:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203708.142183:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203708.142188:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203708.142199:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559911680/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203708.142221:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203708.142284:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203708/real 1788203708] req@ffff93c0e62b8000 x1875064559911424/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203708.142302:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559911424/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203708.142314:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203708.142317:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203708.142321:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559911424:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203708.142351:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559911552:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203708.142372:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203708.142385:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203708/real 1788203708] req@ffff93c0e62b8380 x1875064559911552/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203708.142397:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559911552/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203708.142405:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203708.142408:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203708.142411:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559911552:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203708.142418:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559911680:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203708.142439:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203708.142451:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203708/real 1788203708] req@ffff93c0e62b8700 x1875064559911680/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203708.142463:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559911680/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203708.142472:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203708.142476:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203708.142479:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559911680:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203708.846017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203709.870020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203710.894054:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203711.917996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203712.750049:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203712.750063:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203712.750165:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203712.750204:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203712.750209:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203712.750234:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203712.942011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203713.261953:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203713.261962:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203713.261966:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203713.261976:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2844 00000100:00080000:2.0:1788203713.261987:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203713.261995:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203713.262029:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559911808/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203713.262056:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203713.262059:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203713.262060:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203713.262064:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2844 00000100:00080000:2.0:1788203713.262067:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203713.262070:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203713.262079:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203713.262091:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559911936/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203713.262100:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203713.262102:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559911808:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203713.262105:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203713.262109:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203713.262114:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2844 00000100:00080000:2.0:1788203713.262117:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203713.262120:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203713.262125:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203713.262136:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559912064/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203713.262170:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203713.262229:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203713/real 1788203713] req@ffff93c0e62bb100 x1875064559911808/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203713.262247:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559911808/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203713.262256:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203713.262260:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203713.262264:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559911808:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203713.262285:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559911936:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203713.262305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203713.262316:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203713/real 1788203713] req@ffff93c0e62bbb80 x1875064559911936/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203713.262327:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559911936/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203713.262335:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203713.262338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203713.262341:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559911936:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203713.262346:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559912064:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203713.262361:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203713.262369:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203713/real 1788203713] req@ffff93c0e62bad80 x1875064559912064/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203713.262379:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559912064/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203713.262393:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203713.262396:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203713.262399:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559912064:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203713.966011:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203714.990013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203716.014027:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203717.038039:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203717.869987:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203717.870007:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203717.870125:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203717.870148:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203717.870152:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203717.870173:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203718.062008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203718.381954:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203718.381971:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203718.381975:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203718.381984:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2849 00000100:00080000:0.0:1788203718.381995:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203718.382001:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203718.382043:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559912192/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203718.382064:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203718.382067:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203718.382068:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203718.382071:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2849 00000100:00080000:0.0:1788203718.382073:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203718.382076:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203718.382099:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203718.382112:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559912192:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203718.382118:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559912320/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203718.382126:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203718.382129:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203718.382130:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203718.382134:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2849 00000100:00080000:0.0:1788203718.382136:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203718.382139:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203718.382142:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203718.382151:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559912448/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203718.382184:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203718.382227:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203718/real 1788203718] req@ffff93c0e9ce5180 x1875064559912192/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203718.382244:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559912192/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203718.382259:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203718.382263:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203718.382265:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559912192:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203718.382290:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559912320:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203718.382313:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203718.382326:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203718/real 1788203718] req@ffff93c0e9ce5500 x1875064559912320/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203718.382337:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559912320/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203718.382345:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203718.382347:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203718.382351:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559912320:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203718.382357:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559912448:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203718.382384:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203718.382395:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203718/real 1788203718] req@ffff93c0e9ce7800 x1875064559912448/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203718.382405:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559912448/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203718.382415:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203718.382417:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203718.382421:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559912448:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203719.086016:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203720.109983:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203721.134046:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203722.157983:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203722.990049:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203722.990064:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203722.990161:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203722.990181:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203722.990184:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203722.990199:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203723.181950:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203723.501965:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203723.501973:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203723.501976:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203723.501983:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2854 00000100:00080000:2.0:1788203723.501991:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203723.502017:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203723.502049:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559912576/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203723.502070:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203723.502072:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203723.502073:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203723.502076:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2854 00000100:00080000:2.0:1788203723.502079:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203723.502084:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203723.502091:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203723.502098:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559912704/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203723.502109:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203723.502111:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203723.502113:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203723.502115:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2854 00000100:00080000:2.0:1788203723.502117:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203723.502119:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203723.502121:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203723.502129:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559912832/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203723.502129:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559912576:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203723.502190:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203723.502228:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203723/real 1788203723] req@ffff93c0e62b9500 x1875064559912576/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203723.502243:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559912576/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203723.502253:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203723.502256:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203723.502259:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559912576:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203723.502268:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559912704:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203723.502289:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203723.502301:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203723/real 1788203723] req@ffff93c0e62b9c00 x1875064559912704/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203723.502313:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559912704/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203723.502320:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203723.502323:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203723.502326:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559912704:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203723.502348:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559912832:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203723.502367:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203723.502378:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203723/real 1788203723] req@ffff93c0e62b8a80 x1875064559912832/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203723.502389:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559912832/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203723.502397:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203723.502400:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203723.502403:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559912832:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203724.205969:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203725.230006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203726.254026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203727.278033:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203728.109993:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203728.110010:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203728.110110:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203728.110131:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203728.110139:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203728.110156:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203728.302056:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203728.621896:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203728.621931:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203728.621937:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203728.621950:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2859 00000100:00080000:2.0:1788203728.621964:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203728.621973:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203728.622034:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559912960/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203728.622063:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203728.622066:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203728.622068:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203728.622072:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2859 00000100:00080000:2.0:1788203728.622076:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203728.622080:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203728.622092:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203728.622106:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559912960:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203728.622109:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559913088/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203728.622128:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203728.622131:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203728.622133:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203728.622137:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2859 00000100:00080000:2.0:1788203728.622141:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203728.622145:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203728.622151:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203728.622164:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559913216/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203728.622196:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203728.622245:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203728/real 1788203728] req@ffff93c0e62b9f80 x1875064559912960/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203728.622264:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559912960/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203728.622276:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203728.622281:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203728.622285:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559912960:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203728.622315:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559913088:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203728.622335:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203728.622348:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203728/real 1788203728] req@ffff93c0e62bb800 x1875064559913088/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203728.622361:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559913088/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203728.622369:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203728.622372:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203728.622375:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559913088:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203728.622380:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559913216:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203728.622397:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203728.622407:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203728/real 1788203728] req@ffff93c0e62b8a80 x1875064559913216/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203728.622426:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559913216/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203728.622436:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203728.622438:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203728.622441:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559913216:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203729.326012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203730.350055:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203731.374050:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203732.398075:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203733.229984:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203733.229997:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203733.230094:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203733.230124:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203733.230129:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203733.230161:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203733.421988:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203733.741949:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203733.741957:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203733.741961:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203733.741969:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2864 00000100:00080000:2.0:1788203733.741979:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203733.741985:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203733.742018:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559913344/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203733.742049:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203733.742053:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203733.742056:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203733.742060:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2864 00000100:00080000:2.0:1788203733.742064:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203733.742068:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203733.742079:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203733.742095:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559913472/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203733.742106:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559913344:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203733.742111:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203733.742117:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203733.742119:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203733.742125:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2864 00000100:00080000:2.0:1788203733.742130:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203733.742134:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203733.742140:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203733.742155:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559913600/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203733.742175:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203733.742218:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203733/real 1788203733] req@ffff93c0e62b9c00 x1875064559913344/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203733.742241:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559913344/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203733.742252:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203733.742256:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203733.742261:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559913344:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203733.742284:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559913472:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203733.742303:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203733.742313:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203733/real 1788203733] req@ffff93c0e62b9500 x1875064559913472/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203733.742326:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559913472/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203733.742332:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203733.742334:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203733.742336:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559913472:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203733.742340:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559913600:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203733.742351:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203733.742357:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203733/real 1788203733] req@ffff93c0e62bad80 x1875064559913600/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203733.742365:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559913600/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203733.742382:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203733.742384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203733.742386:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559913600:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203734.446016:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203735.470011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203736.494031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203737.518064:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203738.350034:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203738.350046:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203738.350164:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203738.350197:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203738.350202:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203738.350234:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203738.542043:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203738.861947:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203738.861959:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203738.861963:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203738.861974:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2869 00000100:00080000:0.0:1788203738.861986:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203738.861992:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203738.862029:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559913728/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203738.862064:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203738.862069:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203738.862071:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203738.862076:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2869 00000100:00080000:0.0:1788203738.862080:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203738.862084:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203738.862094:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203738.862109:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559913856/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203738.862123:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559913728:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203738.862124:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203738.862134:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203738.862136:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203738.862141:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2869 00000100:00080000:0.0:1788203738.862146:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203738.862151:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203738.862157:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203738.862172:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559913984/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203738.862191:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203738.862249:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203738/real 1788203738] req@ffff93c0e9ce4e00 x1875064559913728/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203738.862266:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559913728/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203738.862275:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203738.862278:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203738.862281:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559913728:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203738.862302:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559913856:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203738.862320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203738.862329:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203738/real 1788203738] req@ffff93c0e9ce5880 x1875064559913856/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203738.862337:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559913856/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203738.862343:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203738.862345:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203738.862347:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559913856:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203738.862351:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559913984:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203738.862361:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203738.862367:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203738/real 1788203738] req@ffff93c0e9ce6300 x1875064559913984/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203738.862374:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559913984/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203738.862381:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203738.862383:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203738.862385:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559913984:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203739.565992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203740.590083:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203741.613966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203742.638040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203743.470017:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203743.470031:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203743.470120:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203743.470140:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203743.470143:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203743.470159:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203743.662006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203743.981974:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203743.981985:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203743.981989:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203743.982001:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2874 00000100:00080000:0.0:1788203743.982020:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203743.982026:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203743.982061:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559914112/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203743.982084:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203743.982087:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203743.982088:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203743.982091:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2874 00000100:00080000:0.0:1788203743.982094:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203743.982096:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203743.982105:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203743.982115:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559914240/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203743.982130:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203743.982133:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203743.982135:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788203743.982140:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559914112:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203743.982141:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2874 00000100:00080000:0.0:1788203743.982146:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203743.982151:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203743.982155:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203743.982164:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559914368/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203743.982205:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203743.982270:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203743/real 1788203743] req@ffff93c0e9ce6300 x1875064559914112/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203743.982294:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559914112/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203743.982304:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203743.982307:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203743.982310:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559914112:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203743.982329:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559914240:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203743.982348:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203743.982357:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203743/real 1788203743] req@ffff93c0e9ce5880 x1875064559914240/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203743.982365:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559914240/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203743.982373:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203743.982376:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203743.982380:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559914240:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203743.982392:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559914368:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203743.982419:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203743.982432:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203743/real 1788203743] req@ffff93c0e9ce4e00 x1875064559914368/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203743.982445:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559914368/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203743.982454:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203743.982458:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203743.982461:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559914368:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203744.686020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203745.710055:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203746.734035:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203747.758059:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203748.590013:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203748.590027:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203748.590115:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203748.590134:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203748.590137:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203748.590156:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203748.781977:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203749.101938:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203749.101946:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203749.101949:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203749.101958:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2879 00000100:00080000:2.0:1788203749.101967:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203749.101973:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203749.102005:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559914496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203749.102026:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203749.102029:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203749.102030:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203749.102033:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2879 00000100:00080000:2.0:1788203749.102035:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203749.102037:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203749.102045:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203749.102054:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559914624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203749.102061:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203749.102064:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203749.102065:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203749.102067:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2879 00000100:00080000:2.0:1788203749.102069:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788203749.102073:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559914496:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203749.102074:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203749.102082:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203749.102091:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559914752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203749.102132:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203749.102164:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203749/real 1788203749] req@ffff93c0e62bbb80 x1875064559914496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203749.102180:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559914496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203749.102191:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203749.102195:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203749.102199:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559914496:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203749.102207:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559914624:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203749.102232:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203749.102243:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203749/real 1788203749] req@ffff93c0e62bb100 x1875064559914624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203749.102252:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559914624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203749.102258:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203749.102260:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203749.102262:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559914624:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203749.102281:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559914752:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203749.102294:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203749.102300:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203749/real 1788203749] req@ffff93c0e62b8700 x1875064559914752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203749.102308:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559914752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203749.102314:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203749.102316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203749.102318:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559914752:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203749.806049:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203750.830013:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203751.854004:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203752.878032:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203753.710032:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203753.710047:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203753.710137:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203753.710159:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203753.710162:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203753.710180:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203753.902033:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203754.221944:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203754.221953:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203754.221956:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203754.221965:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2885 00000100:00080000:0.0:1788203754.221974:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203754.221981:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203754.222021:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559914880/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203754.222050:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203754.222053:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203754.222054:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203754.222057:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2885 00000100:00080000:0.0:1788203754.222061:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203754.222064:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203754.222073:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203754.222088:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559915008/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203754.222091:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559914880:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203754.222108:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203754.222113:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203754.222115:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203754.222121:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2885 00000100:00080000:0.0:1788203754.222125:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203754.222129:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203754.222135:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203754.222150:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559915136/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203754.222151:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203754.222199:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203754/real 1788203754] req@ffff93c0e9ce7800 x1875064559914880/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203754.222220:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559914880/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203754.222228:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203754.222231:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203754.222234:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559914880:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203754.222252:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559915008:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203754.222267:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203754.222275:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203754/real 1788203754] req@ffff93c0e9ce5500 x1875064559915008/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203754.222283:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559915008/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203754.222289:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203754.222291:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203754.222293:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559915008:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203754.222297:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559915136:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203754.222307:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203754.222313:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203754/real 1788203754] req@ffff93c0e9ce5180 x1875064559915136/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203754.222320:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559915136/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203754.222331:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203754.222333:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203754.222335:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559915136:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203754.926017:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203755.950036:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203756.973985:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203757.998080:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203758.829992:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203758.830011:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203758.830123:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203758.830147:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203758.830152:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203758.830185:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203759.021955:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203759.341944:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203759.341952:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203759.341956:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203759.341964:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2890 00000100:00080000:0.0:1788203759.341982:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203759.341988:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203759.342023:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559915264/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203759.342048:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203759.342050:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203759.342052:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203759.342055:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2890 00000100:00080000:0.0:1788203759.342057:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203759.342059:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203759.342067:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203759.342075:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559915392/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203759.342083:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203759.342085:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203759.342087:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203759.342090:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2890 00000100:00080000:0.0:1788203759.342092:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203759.342099:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203759.342100:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559915264:192.168.201.156@tcp:250:kworker.0 00000008:00100000:0.0:1788203759.342104:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203759.342112:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559915520/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203759.342165:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203759.342206:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203759/real 1788203759] req@ffff93c0e9ce7100 x1875064559915264/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203759.342226:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559915264/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203759.342237:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203759.342242:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203759.342246:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559915264:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203759.342254:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559915392:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203759.342277:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203759.342291:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203759/real 1788203759] req@ffff93c0e9ce4a80 x1875064559915392/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203759.342303:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559915392/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203759.342311:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203759.342315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203759.342318:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559915392:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203759.342341:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559915520:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203759.342360:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203759.342370:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203759/real 1788203759] req@ffff93c0e9ce5f80 x1875064559915520/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203759.342420:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559915520/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203759.342430:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203759.342434:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203759.342437:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559915520:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203760.045969:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203761.069995:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203762.094039:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203763.117974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203763.950107:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203763.950125:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203763.950269:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203763.950331:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203763.950337:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203763.950374:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203764.142003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203764.461998:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203764.462014:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203764.462020:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203764.462033:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2895 00000100:00080000:2.0:1788203764.462049:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203764.462058:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203764.462106:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559915648/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203764.462132:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203764.462136:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203764.462137:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203764.462141:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2895 00000100:00080000:2.0:1788203764.462144:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203764.462147:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203764.462157:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203764.462169:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559915776/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203764.462177:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203764.462180:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203764.462181:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203764.462185:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2895 00000100:00080000:2.0:1788203764.462187:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203764.462189:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203764.462193:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203764.462196:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559915648:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203764.462203:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559915904/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203764.462304:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203764.462371:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203764/real 1788203764] req@ffff93c0e62b8380 x1875064559915648/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203764.462396:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559915648/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203764.462409:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203764.462413:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203764.462418:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559915648:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203764.462427:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559915776:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203764.462452:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203764.462465:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203764/real 1788203764] req@ffff93c0e62b8000 x1875064559915776/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203764.462479:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559915776/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203764.462488:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203764.462492:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203764.462495:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559915776:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203764.462524:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559915904:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203764.462547:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203764.462559:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203764/real 1788203764] req@ffff93c0e62ba680 x1875064559915904/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203764.462573:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559915904/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203764.462584:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203764.462588:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203764.462591:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559915904:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203765.166037:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203766.190021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203767.214054:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203768.237973:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203769.069993:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203769.070006:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203769.070096:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203769.070118:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203769.070121:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203769.070145:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203769.262036:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203769.582036:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203769.582045:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203769.582048:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203769.582056:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2900 00000100:00080000:3.0:1788203769.582065:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203769.582071:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203769.582105:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559916032/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203769.582132:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203769.582135:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203769.582137:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203769.582140:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2900 00000100:00080000:3.0:1788203769.582142:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203769.582144:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203769.582152:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203769.582160:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559916160/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203769.582168:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203769.582171:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203769.582172:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203769.582177:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2900 00000100:00080000:3.0:1788203769.582179:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203769.582182:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203769.582185:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203769.582193:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559916288/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203769.582228:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559916032:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203769.582335:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559916160:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203769.582378:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559916288:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203770.286052:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203771.310051:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203772.334033:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788203772.654191:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788203772.654248:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788203772.654271:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788203772.654334:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788203772.654346:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788203772.654402:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203769/real 1788203772] req@ffff93c1c4a4ed80 x1875064559916288/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203772.654435:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559916288/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203772.654453:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203772.654459:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203772.654465:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559916288:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788203772.654475:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203769/real 1788203772] req@ffff93c1c4a4dc00 x1875064559916032/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203772.654490:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559916032/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203772.654499:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203772.654504:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203772.654508:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559916032:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203772.654517:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203769/real 1788203772] req@ffff93c1c4a4e680 x1875064559916160/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203772.654531:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559916160/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203772.654546:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203772.654550:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203772.654554:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559916160:192.168.201.156@tcp:38:kworker.0 00010000:00010000:2.0:1788203773.358018:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203774.189974:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203774.189991:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203774.190086:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203774.190108:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203774.190111:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203774.190148:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203774.382072:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203774.701943:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203774.701955:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203774.701960:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203774.701972:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2905 00000100:00080000:3.0:1788203774.701984:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203774.701992:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203774.702036:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4df80 x1875064559916416/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203774.702085:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203774.702089:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203774.702093:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203774.702098:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2905 00000100:00080000:3.0:1788203774.702102:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203774.702120:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203774.702133:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203774.702150:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f100 x1875064559916544/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203774.702163:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203774.702167:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203774.702169:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203774.702176:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2905 00000100:00080000:3.0:1788203774.702180:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203774.702186:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203774.702191:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559916416:192.168.201.156@tcp:250:kworker.0 00000008:00100000:3.0:1788203774.702194:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203774.702231:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559916672/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203774.702290:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203774.702366:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203774/real 1788203774] req@ffff93c1c4a4df80 x1875064559916416/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203774.702388:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4df80 x1875064559916416/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203774.702401:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203774.702406:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203774.702411:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559916416:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203774.702420:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559916544:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203774.702449:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203774.702464:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203774/real 1788203774] req@ffff93c1c4a4f100 x1875064559916544/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203774.702475:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f100 x1875064559916544/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203774.702484:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203774.702487:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203774.702490:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559916544:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203774.702532:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559916672:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203774.702556:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203774.702568:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203774/real 1788203774] req@ffff93c1c4a4e680 x1875064559916672/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203774.702581:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559916672/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203774.702591:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203774.702594:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203774.702597:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559916672:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203775.406040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203776.430000:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203777.454036:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203778.478017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203779.310004:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203779.310022:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203779.310126:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203779.310149:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203779.310160:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203779.310181:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203779.501998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203779.821993:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203779.822005:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203779.822010:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203779.822054:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2910 00000100:00080000:3.0:1788203779.822067:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203779.822074:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203779.822112:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559916800/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203779.822147:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203779.822156:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203779.822159:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203779.822163:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2910 00000100:00080000:3.0:1788203779.822168:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203779.822172:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203779.822183:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203779.822200:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559916800:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203779.822200:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559916928/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203779.822218:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203779.822222:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203779.822224:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203779.822228:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2910 00000100:00080000:3.0:1788203779.822232:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203779.822236:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203779.822242:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203779.822254:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559917056/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203779.822275:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203779.822332:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203779/real 1788203779] req@ffff93c1c4a4dc00 x1875064559916800/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203779.822354:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559916800/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203779.822366:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203779.822371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203779.822375:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559916800:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203779.822400:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559916928:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203779.822426:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203779.822440:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203779/real 1788203779] req@ffff93c1c4a4ed80 x1875064559916928/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203779.822455:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559916928/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203779.822464:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203779.822468:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203779.822471:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559916928:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203779.822478:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559917056:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203779.822496:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203779.822504:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203779/real 1788203779] req@ffff93c1c4a4fb80 x1875064559917056/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203779.822512:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559917056/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203779.822519:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203779.822521:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203779.822523:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559917056:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203780.525972:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203781.550042:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203782.574062:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203783.598074:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203784.429991:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203784.430013:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203784.430136:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203784.430168:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203784.430173:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203784.430201:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203784.622116:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203784.941915:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203784.941955:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203784.941962:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203784.941975:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2915 00000100:00080000:3.0:1788203784.941988:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203784.941996:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203784.942040:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559917184/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203784.942068:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203784.942071:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203784.942072:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203784.942076:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2915 00000100:00080000:3.0:1788203784.942079:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203784.942081:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203784.942091:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203784.942101:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559917312/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203784.942110:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203784.942112:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203784.942114:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203784.942117:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2915 00000100:00080000:3.0:1788203784.942119:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203784.942121:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203784.942125:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203784.942129:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559917184:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203784.942135:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559917440/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203784.942225:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203784.942277:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203784/real 1788203784] req@ffff93c1c4a4c000 x1875064559917184/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203784.942294:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559917184/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203784.942313:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203784.942318:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203784.942325:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559917184:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203784.942335:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559917312:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203784.942364:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203784.942380:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203784/real 1788203784] req@ffff93c1c4a4ea00 x1875064559917312/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203784.942396:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559917312/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203784.942403:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203784.942406:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203784.942408:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559917312:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203784.942437:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559917440:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203784.942463:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203784.942475:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203784/real 1788203784] req@ffff93c1c4a4f480 x1875064559917440/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203784.942484:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559917440/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203784.942492:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203784.942495:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203784.942497:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559917440:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203785.646045:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203786.670080:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203787.694019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203788.718050:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203789.550013:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203789.550033:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203789.550173:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203789.550208:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203789.550214:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203789.550253:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203789.742054:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203790.062006:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203790.062022:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203790.062027:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203790.062040:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2920 00000100:00080000:3.0:1788203790.062054:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203790.062063:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203790.062118:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559917568/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203790.062148:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203790.062151:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203790.062159:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203790.062163:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2920 00000100:00080000:3.0:1788203790.062167:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203790.062171:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203790.062182:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203790.062192:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559917696/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203790.062201:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203790.062204:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203790.062206:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788203790.062212:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559917568:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203790.062213:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2920 00000100:00080000:3.0:1788203790.062217:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203790.062224:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203790.062228:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203790.062238:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559917824/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203790.062307:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203790.062387:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203790/real 1788203790] req@ffff93c1c4a4c380 x1875064559917568/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203790.062411:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559917568/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203790.062425:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203790.062431:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203790.062435:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559917568:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203790.062464:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559917696:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203790.062486:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203790.062496:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203790/real 1788203790] req@ffff93c1c4a4c700 x1875064559917696/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203790.062504:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559917696/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203790.062511:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203790.062513:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203790.062516:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559917696:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203790.062519:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559917824:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203790.062532:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203790.062538:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203790/real 1788203790] req@ffff93c1c4a4e300 x1875064559917824/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203790.062547:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559917824/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203790.062554:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203790.062556:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203790.062558:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559917824:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203790.766077:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203791.790048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203792.814055:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203793.838058:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203794.670013:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203794.670028:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203794.670133:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203794.670165:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203794.670170:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203794.670202:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203794.862040:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203795.181939:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203795.181950:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203795.181954:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203795.181964:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2926 00000100:00080000:3.0:1788203795.181975:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203795.181982:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203795.182020:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559917952/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203795.182051:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203795.182056:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203795.182066:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203795.182071:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2926 00000100:00080000:3.0:1788203795.182075:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203795.182079:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203795.182090:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203795.182103:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559918080/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203795.182116:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788203795.182121:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559917952:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203795.182122:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203795.182126:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203795.182134:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2926 00000100:00080000:3.0:1788203795.182137:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203795.182142:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203795.182147:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203795.182161:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559918208/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203795.182186:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203795.182221:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203795/real 1788203795] req@ffff93c1c4a4d180 x1875064559917952/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203795.182235:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559917952/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203795.182242:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203795.182245:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203795.182248:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559917952:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203795.182266:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559918080:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203795.182280:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203795.182287:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203795/real 1788203795] req@ffff93c1c4a4f800 x1875064559918080/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203795.182295:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559918080/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203795.182305:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203795.182307:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203795.182310:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559918080:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203795.182313:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559918208:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203795.182324:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203795.182330:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203795/real 1788203795] req@ffff93c1c4a4d880 x1875064559918208/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203795.182336:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559918208/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203795.182352:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203795.182355:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203795.182357:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559918208:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203795.886031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203796.910054:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203797.933988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203798.958041:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203799.789999:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203799.790020:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203799.790149:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203799.790180:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203799.790185:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203799.790213:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203799.982044:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203800.301949:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203800.301963:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203800.301968:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203800.301979:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2931 00000100:00080000:3.0:1788203800.301991:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203800.301998:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203800.302039:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d500 x1875064559918336/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203800.302064:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203800.302067:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203800.302068:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203800.302072:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2931 00000100:00080000:3.0:1788203800.302074:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203800.302076:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203800.302086:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203800.302094:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ca80 x1875064559918464/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203800.302102:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203800.302105:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203800.302106:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203800.302109:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2931 00000100:00080000:3.0:1788203800.302111:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203800.302113:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203800.302116:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203800.302122:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559918336:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203800.302125:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559918592/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203800.302208:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203800.302267:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203800/real 1788203800] req@ffff93c1c4a4d500 x1875064559918336/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203800.302286:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d500 x1875064559918336/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203800.302297:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203800.302301:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203800.302306:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559918336:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203800.302316:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559918464:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203800.302338:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203800.302367:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203800/real 1788203800] req@ffff93c1c4a4ca80 x1875064559918464/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203800.302379:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ca80 x1875064559918464/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203800.302386:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203800.302389:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203800.302392:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559918464:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203800.302417:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559918592:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203800.302436:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203800.302446:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203800/real 1788203800] req@ffff93c1c4a4d880 x1875064559918592/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203800.302462:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559918592/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203800.302473:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203800.302476:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203800.302479:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559918592:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203801.006040:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203802.029985:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203803.054062:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203804.078045:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203804.910052:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203804.910069:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203804.910189:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203804.910209:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203804.910212:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203804.910230:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203805.101989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203805.421975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203805.421985:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203805.421988:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203805.421999:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2936 00000100:00080000:2.0:1788203805.422011:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203805.422017:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203805.422058:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559918720/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203805.422090:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203805.422093:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203805.422095:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203805.422098:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2936 00000100:00080000:2.0:1788203805.422100:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203805.422102:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203805.422112:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203805.422120:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559918848/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203805.422126:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203805.422129:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203805.422130:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203805.422134:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2936 00000100:00080000:2.0:1788203805.422135:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203805.422140:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203805.422141:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559918720:192.168.201.156@tcp:250:kworker.0 00000008:00100000:2.0:1788203805.422143:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203805.422155:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559918976/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203805.422212:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203805.422252:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203805/real 1788203805] req@ffff93c0e62baa00 x1875064559918720/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203805.422265:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559918720/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203805.422273:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203805.422276:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203805.422279:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559918720:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203805.422284:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559918848:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203805.422297:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203805.422304:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203805/real 1788203805] req@ffff93c0e62bb480 x1875064559918848/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203805.422312:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559918848/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203805.422317:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203805.422319:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203805.422321:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559918848:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203805.422348:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559918976:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203805.422363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203805.422370:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203805/real 1788203805] req@ffff93c0e62ba680 x1875064559918976/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203805.422377:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559918976/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203805.422388:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203805.422390:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203805.422392:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559918976:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203806.126002:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203807.149971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203808.174036:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203809.198021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203810.030022:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203810.030041:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203810.030181:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203810.030206:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203810.030211:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203810.030241:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203810.222037:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203810.541915:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203810.541926:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203810.541930:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203810.541941:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2941 00000100:00080000:2.0:1788203810.541952:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203810.541959:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203810.541999:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559919104/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203810.542024:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203810.542027:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203810.542028:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203810.542031:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2941 00000100:00080000:2.0:1788203810.542033:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203810.542036:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203810.542045:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203810.542052:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559919232/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203810.542059:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203810.542062:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203810.542064:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203810.542068:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2941 00000100:00080000:2.0:1788203810.542070:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203810.542072:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203810.542076:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203810.542082:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559919360/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203810.542093:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559919104:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788203810.542171:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203810.542229:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203810/real 1788203810] req@ffff93c0e62b8000 x1875064559919104/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203810.542249:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559919104/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203810.542260:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203810.542263:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203810.542267:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559919104:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203810.542274:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559919232:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203810.542295:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203810.542306:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203810/real 1788203810] req@ffff93c0e62b8380 x1875064559919232/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203810.542318:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559919232/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203810.542326:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203810.542329:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203810.542333:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559919232:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203810.542360:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559919360:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203810.542379:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203810.542391:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203810/real 1788203810] req@ffff93c0e62b8700 x1875064559919360/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203810.542401:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559919360/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203810.542411:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203810.542414:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203810.542418:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559919360:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203811.246006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203812.270018:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203813.294035:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203814.318048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203815.150006:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203815.150019:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203815.150119:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203815.150146:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203815.150151:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203815.150177:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203815.342010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203815.661975:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203815.661985:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203815.661990:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203815.662001:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2946 00000100:00080000:3.0:1788203815.662013:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203815.662020:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203815.662069:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559919488/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203815.662101:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203815.662104:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203815.662106:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203815.662111:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2946 00000100:00080000:3.0:1788203815.662115:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203815.662121:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203815.662133:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203815.662146:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559919616/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203815.662154:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203815.662159:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203815.662161:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788203815.662162:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559919488:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203815.662171:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2946 00000100:00080000:3.0:1788203815.662176:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203815.662181:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203815.662185:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203815.662195:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559919744/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203815.662235:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203815.662282:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203815/real 1788203815] req@ffff93c1c4a4f800 x1875064559919488/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203815.662301:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559919488/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203815.662312:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203815.662317:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203815.662321:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559919488:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203815.662345:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559919616:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203815.662368:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203815.662381:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203815/real 1788203815] req@ffff93c1c4a4d180 x1875064559919616/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203815.662394:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559919616/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203815.662414:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203815.662417:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203815.662419:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559919616:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203815.662425:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559919744:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203815.662440:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203815.662448:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203815/real 1788203815] req@ffff93c1c4a4e300 x1875064559919744/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203815.662455:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559919744/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203815.662461:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203815.662463:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203815.662465:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559919744:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203816.366041:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203817.389998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203818.414055:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203819.437984:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203820.270023:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203820.270043:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203820.270162:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203820.270185:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203820.270188:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203820.270206:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203820.462029:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203820.781969:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203820.781981:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203820.781986:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203820.781999:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2951 00000100:00080000:0.0:1788203820.782012:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203820.782021:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203820.782074:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559919872/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203820.782099:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203820.782103:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203820.782104:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203820.782109:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2951 00000100:00080000:0.0:1788203820.782113:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203820.782117:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203820.782129:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203820.782141:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559920000/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203820.782142:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559919872:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203820.782151:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203820.782154:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203820.782155:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203820.782159:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2951 00000100:00080000:0.0:1788203820.782162:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203820.782165:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203820.782169:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203820.782177:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559920128/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203820.782216:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203820.782265:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203820/real 1788203820] req@ffff93c0e9ce6d80 x1875064559919872/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203820.782280:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559919872/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203820.782288:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203820.782292:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203820.782295:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559919872:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203820.782333:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559920000:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203820.782351:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203820.782360:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203820/real 1788203820] req@ffff93c0e9ce5c00 x1875064559920000/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203820.782367:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559920000/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203820.782377:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203820.782379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203820.782381:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559920000:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203820.782384:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559920128:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203820.782394:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203820.782399:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203820/real 1788203820] req@ffff93c0e9ce7b80 x1875064559920128/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203820.782406:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559920128/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203820.782413:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203820.782414:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203820.782416:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559920128:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788203821.486018:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203822.509999:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788203823.534048:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203824.557998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203825.389961:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203825.389976:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203825.390092:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203825.390125:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203825.390130:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203825.390163:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788203825.582009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203825.901965:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203825.901982:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203825.901989:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203825.901999:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2956 00000100:00080000:3.0:1788203825.902009:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203825.902015:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203825.902050:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559920256/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203825.902078:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203825.902081:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203825.902083:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203825.902088:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2956 00000100:00080000:3.0:1788203825.902091:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203825.902095:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203825.902105:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203825.902107:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559920256:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203825.902123:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559920384/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203825.902134:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203825.902138:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203825.902140:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203825.902143:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2956 00000100:00080000:3.0:1788203825.902147:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203825.902150:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203825.902155:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203825.902167:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559920512/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203825.902173:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203825.902237:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203825/real 1788203825] req@ffff93c1c4a4c700 x1875064559920256/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203825.902260:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559920256/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203825.902269:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203825.902273:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203825.902276:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559920256:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203825.902297:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559920384:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203825.902318:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203825.902330:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203825/real 1788203825] req@ffff93c1c4a4c380 x1875064559920384/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203825.902340:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559920384/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203825.902345:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203825.902347:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203825.902349:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559920384:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203825.902354:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559920512:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203825.902373:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203825.902384:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203825/real 1788203825] req@ffff93c1c4a4f480 x1875064559920512/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203825.902395:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559920512/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203825.902410:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203825.902412:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203825.902414:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559920512:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203826.605977:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203827.630033:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203828.653990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203829.677976:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203830.509996:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203830.510012:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203830.510108:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203830.510131:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203830.510135:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203830.510182:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203830.702010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203831.021888:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203831.021898:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203831.021902:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203831.021911:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2961 00000100:00080000:3.0:1788203831.021921:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203831.021927:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203831.021957:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559920640/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203831.021988:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203831.021991:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203831.021993:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203831.021997:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2961 00000100:00080000:3.0:1788203831.021999:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203831.022001:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203831.022008:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203831.022017:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559920768/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203831.022028:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203831.022032:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788203831.022033:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559920640:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203831.022034:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203831.022042:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2961 00000100:00080000:3.0:1788203831.022044:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203831.022046:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203831.022049:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203831.022057:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559920896/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203831.022087:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203831.022115:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203831/real 1788203831] req@ffff93c1c4a4ea00 x1875064559920640/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203831.022127:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559920640/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203831.022147:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203831.022150:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203831.022153:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559920640:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203831.022170:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559920768:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203831.022187:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203831.022195:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203831/real 1788203831] req@ffff93c1c4a4c000 x1875064559920768/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203831.022202:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559920768/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203831.022207:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203831.022209:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203831.022211:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559920768:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203831.022215:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559920896:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203831.022225:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203831.022231:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203831/real 1788203831] req@ffff93c1c4a4fb80 x1875064559920896/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203831.022237:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559920896/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203831.022243:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203831.022245:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203831.022246:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559920896:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203831.725992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203832.750027:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203833.773994:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203834.798022:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203835.629993:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203835.630007:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203835.630127:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203835.630148:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203835.630152:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203835.630180:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203835.821986:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203836.141947:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203836.141954:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203836.141957:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203836.141965:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2967 00000100:00080000:1.0:1788203836.141974:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203836.141980:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203836.142013:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559921024/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203836.142056:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203836.142060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203836.142062:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203836.142067:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2967 00000100:00080000:1.0:1788203836.142071:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203836.142075:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203836.142082:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559921024:192.168.201.156@tcp:250:kworker.0 00000008:00100000:1.0:1788203836.142084:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203836.142102:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559921152/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203836.142113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203836.142116:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203836.142118:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203836.142122:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2967 00000100:00080000:1.0:1788203836.142126:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203836.142130:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203836.142135:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203836.142146:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559921280/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203836.142195:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559921152:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203836.142232:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559921280:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203836.846004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203837.869986:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203838.893949:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788203839.214185:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788203839.214242:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788203839.214263:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788203839.214321:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788203839.214333:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788203839.214435:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203836/real 1788203839] req@ffff93c1c5036680 x1875064559921152/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203839.214471:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559921152/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203839.214492:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203839.214498:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203839.214503:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559921152:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203839.214512:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203836/real 1788203839] req@ffff93c1c5035c00 x1875064559921280/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203839.214525:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559921280/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203839.214535:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203839.214538:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203839.214542:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559921280:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788203839.214549:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203836/real 1788203839] req@ffff93c1c5037800 x1875064559921024/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203839.214576:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559921024/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203839.214583:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203839.214586:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203839.214589:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559921024:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788203839.917988:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203840.750045:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203840.750062:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203840.750158:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203840.750180:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203840.750183:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203840.750198:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203840.942028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203841.262040:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203841.262053:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203841.262057:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203841.262070:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2972 00000100:00080000:3.0:1788203841.262083:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203841.262093:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203841.262136:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559921408/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203841.262171:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203841.262175:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203841.262177:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203841.262182:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2972 00000100:00080000:3.0:1788203841.262186:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203841.262190:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203841.262202:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203841.262218:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559921536/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203841.262228:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203841.262232:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203841.262234:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788203841.262241:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559921408:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203841.262242:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2972 00000100:00080000:3.0:1788203841.262247:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203841.262253:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203841.262259:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203841.262273:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559921664/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203841.262339:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203841.262394:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203841/real 1788203841] req@ffff93c1c4a4ed80 x1875064559921408/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203841.262416:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559921408/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203841.262428:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203841.262433:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203841.262438:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559921408:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203841.262468:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559921536:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203841.262493:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203841.262506:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203841/real 1788203841] req@ffff93c1c4a4dc00 x1875064559921536/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203841.262519:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559921536/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203841.262526:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203841.262529:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203841.262532:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559921536:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203841.262538:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559921664:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203841.262557:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203841.262569:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203841/real 1788203841] req@ffff93c1c4a4e680 x1875064559921664/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203841.262581:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559921664/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203841.262597:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203841.262601:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203841.262608:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559921664:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203841.965980:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203842.990046:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203844.013980:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203845.038020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203845.870066:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203845.870080:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203845.870166:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203845.870185:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203845.870187:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203845.870209:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203846.061995:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203846.381977:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203846.381986:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203846.381989:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203846.381999:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2977 00000100:00080000:3.0:1788203846.382009:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203846.382015:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203846.382049:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f100 x1875064559921792/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203846.382072:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203846.382075:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203846.382077:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203846.382080:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2977 00000100:00080000:3.0:1788203846.382083:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203846.382086:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203846.382095:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203846.382104:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4df80 x1875064559921920/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203846.382114:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788203846.382117:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559921792:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203846.382118:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203846.382122:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203846.382128:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2977 00000100:00080000:3.0:1788203846.382131:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203846.382134:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203846.382138:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203846.382148:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559922048/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203846.382182:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203846.382224:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203846/real 1788203846] req@ffff93c1c4a4f100 x1875064559921792/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203846.382242:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f100 x1875064559921792/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203846.382252:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203846.382256:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203846.382259:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559921792:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203846.382277:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559921920:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203846.382293:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203846.382301:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203846/real 1788203846] req@ffff93c1c4a4df80 x1875064559921920/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203846.382308:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4df80 x1875064559921920/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203846.382317:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203846.382319:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203846.382321:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559921920:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203846.382325:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559922048:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203846.382341:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203846.382352:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203846/real 1788203846] req@ffff93c1c4a4e680 x1875064559922048/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203846.382362:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559922048/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203846.382370:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203846.382373:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203846.382376:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559922048:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203847.086041:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203848.109976:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203849.133971:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203850.157939:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203850.990020:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203850.990036:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203850.990131:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203850.990150:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203850.990152:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203850.990168:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203851.182027:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203851.501911:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203851.501923:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203851.501928:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203851.501940:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2982 00000100:00080000:3.0:1788203851.501952:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203851.501960:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203851.502003:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559922176/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203851.502033:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203851.502036:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203851.502038:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203851.502043:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2982 00000100:00080000:3.0:1788203851.502046:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203851.502050:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203851.502062:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203851.502074:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559922304/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203851.502086:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559922176:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203851.502101:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203851.502105:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203851.502108:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203851.502113:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2982 00000100:00080000:3.0:1788203851.502116:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203851.502120:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203851.502125:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203851.502138:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559922432/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203851.502179:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203851.502247:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203851/real 1788203851] req@ffff93c1c4a4dc00 x1875064559922176/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203851.502271:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559922176/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203851.502282:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203851.502287:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203851.502291:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559922176:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203851.502322:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559922304:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203851.502348:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203851.502359:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203851/real 1788203851] req@ffff93c1c4a4ed80 x1875064559922304/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203851.502377:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559922304/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203851.502385:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203851.502388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203851.502391:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559922304:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203851.502405:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559922432:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203851.502426:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203851.502445:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203851/real 1788203851] req@ffff93c1c4a4fb80 x1875064559922432/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203851.502461:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559922432/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203851.502477:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203851.502481:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203851.502488:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559922432:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203852.206020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203853.230042:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203854.253953:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203855.278102:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203856.110057:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203856.110077:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203856.110222:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203856.110252:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203856.110257:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203856.110292:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203856.301973:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203856.621965:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203856.621976:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203856.621980:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203856.621991:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2987 00000100:00080000:0.0:1788203856.622003:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203856.622011:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203856.622058:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559922560/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203856.622086:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203856.622089:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203856.622091:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203856.622096:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2987 00000100:00080000:0.0:1788203856.622100:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203856.622104:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203856.622116:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203856.622126:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559922560:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203856.622128:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559922688/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203856.622137:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203856.622140:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203856.622142:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203856.622146:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2987 00000100:00080000:0.0:1788203856.622148:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203856.622152:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203856.622156:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203856.622166:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559922816/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203856.622196:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203856.622244:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203856/real 1788203856] req@ffff93c0e9ce6680 x1875064559922560/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203856.622265:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559922560/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203856.622277:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203856.622281:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203856.622286:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559922560:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203856.622313:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559922688:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203856.622334:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203856.622345:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203856/real 1788203856] req@ffff93c0e9ce4000 x1875064559922688/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203856.622357:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559922688/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203856.622365:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203856.622369:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203856.622372:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559922688:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203856.622378:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559922816:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203856.622397:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203856.622408:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203856/real 1788203856] req@ffff93c0e9ce6a00 x1875064559922816/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203856.622420:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559922816/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203856.622430:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203856.622432:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203856.622435:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559922816:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203857.325981:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203858.349914:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203859.374020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203860.398029:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203861.230004:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203861.230022:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203861.230128:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203861.230156:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203861.230159:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203861.230176:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203861.421966:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203861.741969:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203861.741981:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203861.741985:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203861.741995:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2992 00000100:00080000:1.0:1788203861.742007:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203861.742014:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203861.742054:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559922944/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203861.742082:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203861.742085:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203861.742086:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203861.742089:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2992 00000100:00080000:1.0:1788203861.742092:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203861.742094:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203861.742103:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203861.742112:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559923072/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203861.742120:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203861.742122:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203861.742123:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203861.742128:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2992 00000100:00080000:1.0:1788203861.742129:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203861.742131:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203861.742136:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203861.742144:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559923200/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203861.742144:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559922944:192.168.201.156@tcp:250:kworker.0 00000800:00000100:3.0:1788203861.742229:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203861.742277:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203861/real 1788203861] req@ffff93c1c5037800 x1875064559922944/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203861.742300:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559922944/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203861.742311:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203861.742316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203861.742320:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559922944:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203861.742345:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559923072:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203861.742362:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203861.742370:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203861/real 1788203861] req@ffff93c1c5035c00 x1875064559923072/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203861.742378:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559923072/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203861.742384:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203861.742386:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203861.742388:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559923072:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203861.742392:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559923200:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203861.742402:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203861.742407:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203861/real 1788203861] req@ffff93c1c5036680 x1875064559923200/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203861.742419:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559923200/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203861.742426:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203861.742427:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203861.742429:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559923200:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203862.445996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203863.470005:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203864.493991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203865.517959:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203866.350037:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203866.350058:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203866.350167:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203866.350186:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203866.350189:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203866.350205:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203866.541939:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203866.861973:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203866.861987:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203866.861992:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203866.862005:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 2997 00000100:00080000:2.0:1788203866.862019:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203866.862027:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203866.862074:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559923328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203866.862103:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203866.862107:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203866.862108:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203866.862114:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2997 00000100:00080000:2.0:1788203866.862118:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203866.862123:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203866.862134:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203866.862148:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559923456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203866.862160:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788203866.862162:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559923328:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203866.862167:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203866.862171:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203866.862176:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 2997 00000100:00080000:2.0:1788203866.862180:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203866.862183:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203866.862189:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203866.862199:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559923584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203866.862235:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203866.862281:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203866/real 1788203866] req@ffff93c0e62bb100 x1875064559923328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203866.862297:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559923328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203866.862306:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203866.862309:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203866.862312:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559923328:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203866.862331:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559923456:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203866.862346:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203866.862353:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203866/real 1788203866] req@ffff93c0e62bbb80 x1875064559923456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203866.862360:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559923456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203866.862369:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203866.862371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203866.862373:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559923456:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203866.862376:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559923584:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203866.862386:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203866.862390:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203866/real 1788203866] req@ffff93c0e62bad80 x1875064559923584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203866.862397:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559923584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203866.862403:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203866.862405:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203866.862406:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559923584:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788203867.565992:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203868.589977:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203869.614028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203870.637984:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203871.469975:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203871.469993:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203871.470111:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203871.470135:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203871.470138:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203871.470164:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788203871.662046:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203871.981956:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203871.981972:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203871.981977:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203871.981991:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3002 00000100:00080000:2.0:1788203871.982006:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203871.982015:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203871.982067:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559923712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203871.982099:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203871.982102:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203871.982103:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203871.982110:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3002 00000100:00080000:2.0:1788203871.982113:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203871.982116:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203871.982126:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203871.982137:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559923840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203871.982145:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203871.982148:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203871.982149:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203871.982154:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3002 00000100:00080000:2.0:1788203871.982157:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788203871.982157:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559923712:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203871.982162:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203871.982167:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203871.982174:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559923968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203871.982252:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203871.982306:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203871/real 1788203871] req@ffff93c0e62b9500 x1875064559923712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203871.982330:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559923712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203871.982342:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203871.982346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203871.982351:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559923712:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203871.982380:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559923840:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203871.982407:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203871.982429:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203871/real 1788203871] req@ffff93c0e62b9c00 x1875064559923840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203871.982439:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559923840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203871.982449:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203871.982451:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203871.982454:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559923840:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203871.982458:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559923968:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203871.982472:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203871.982479:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203871/real 1788203871] req@ffff93c0e62b8a80 x1875064559923968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203871.982488:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559923968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203871.982498:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203871.982501:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203871.982504:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559923968:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203872.686018:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788203873.710044:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203874.734012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203875.758043:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203876.590058:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203876.590075:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203876.591513:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203876.591559:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203876.591565:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203876.591596:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203876.782020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203877.101993:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203877.102005:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203877.102008:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203877.102018:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3007 00000100:00080000:3.0:1788203877.102029:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203877.102036:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203877.102074:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559924096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203877.102111:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203877.102115:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203877.102117:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203877.102121:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3007 00000100:00080000:3.0:1788203877.102124:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203877.102129:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203877.102139:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203877.102153:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559924224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203877.102164:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203877.102170:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788203877.102173:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559924096:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203877.102173:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203877.102181:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3007 00000100:00080000:3.0:1788203877.102191:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203877.102195:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203877.102200:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203877.102213:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559924352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203877.103379:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203877.103419:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203877/real 1788203877] req@ffff93c1c4a4c000 x1875064559924096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203877.103432:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559924096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203877.103439:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203877.103442:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203877.103445:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559924096:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203877.103469:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559924224:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203877.103486:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203877.103494:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203877/real 1788203877] req@ffff93c1c4a4ea00 x1875064559924224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203877.103501:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559924224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203877.103506:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203877.103508:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203877.103509:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559924224:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203877.103513:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559924352:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203877.103523:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203877.103528:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203877/real 1788203877] req@ffff93c1c4a4f480 x1875064559924352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203877.103539:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559924352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203877.103545:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203877.103546:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203877.103548:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559924352:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203877.806008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203878.829998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203879.854098:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203880.877992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203881.709967:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203881.709996:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203881.710092:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203881.710113:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203881.710116:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203881.710134:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203881.902004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203882.221981:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203882.221993:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203882.221998:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203882.222009:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3013 00000100:00080000:3.0:1788203882.222020:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203882.222027:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203882.222065:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559924480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203882.222097:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203882.222101:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203882.222103:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203882.222107:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3013 00000100:00080000:3.0:1788203882.222111:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203882.222114:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203882.222124:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203882.222137:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559924608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203882.222148:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203882.222152:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203882.222156:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788203882.222158:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559924480:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203882.222163:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3013 00000100:00080000:3.0:1788203882.222171:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203882.222175:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203882.222180:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203882.222192:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559924736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203882.222235:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203882.222336:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203882/real 1788203882] req@ffff93c1c4a4c380 x1875064559924480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203882.222351:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559924480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203882.222360:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203882.222364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203882.222367:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559924480:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203882.222387:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559924608:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203882.222412:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203882.222426:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203882/real 1788203882] req@ffff93c1c4a4c700 x1875064559924608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203882.222440:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559924608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203882.222448:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203882.222451:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203882.222455:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559924608:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203882.222461:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559924736:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203882.222484:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203882.222500:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203882/real 1788203882] req@ffff93c1c4a4e300 x1875064559924736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203882.222512:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559924736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203882.222519:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203882.222522:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203882.222524:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559924736:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203882.925996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203883.950006:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203884.974000:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203885.997993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203886.831047:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203886.831060:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203886.831155:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203886.831190:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203886.831193:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203886.831213:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203887.022011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203887.341978:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203887.341987:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203887.341990:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203887.341999:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3018 00000100:00080000:1.0:1788203887.342008:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203887.342014:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203887.342047:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559924864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203887.342069:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203887.342072:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203887.342074:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203887.342077:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3018 00000100:00080000:1.0:1788203887.342079:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203887.342082:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203887.342089:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203887.342099:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559924992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203887.342107:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203887.342110:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203887.342111:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203887.342114:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3018 00000100:00080000:1.0:1788203887.342116:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203887.342119:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203887.342122:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203887.342132:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559925120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203887.342134:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559924864:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788203887.342220:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203887.342263:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203887/real 1788203887] req@ffff93c1c5034e00 x1875064559924864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203887.342283:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559924864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203887.342293:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203887.342297:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203887.342301:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559924864:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203887.342309:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559924992:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203887.342330:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203887.342463:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203887/real 1788203887] req@ffff93c1c5034a80 x1875064559924992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203887.342487:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559924992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203887.342501:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203887.342504:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203887.342507:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559924992:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203887.342540:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559925120:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203887.342593:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203887.342611:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203887/real 1788203887] req@ffff93c1c5037100 x1875064559925120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203887.342619:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559925120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203887.342632:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203887.342634:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203887.342636:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559925120:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203888.046010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203889.070007:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203890.095232:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203891.118013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203891.951322:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203891.951337:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203891.951452:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203891.951481:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203891.951485:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203891.951504:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203892.142052:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203892.461900:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203892.461916:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203892.461922:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203892.461935:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3023 00000100:00080000:1.0:1788203892.461950:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203892.461960:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203892.462015:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559925248/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203892.462044:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203892.462047:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203892.462049:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203892.462053:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3023 00000100:00080000:1.0:1788203892.462056:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203892.462059:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203892.462081:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203892.462094:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559925376/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203892.462104:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203892.462107:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203892.462109:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788203892.462115:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559925248:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788203892.462115:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3023 00000100:00080000:1.0:1788203892.462121:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203892.462126:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203892.462130:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203892.462137:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559925504/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203892.462203:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203892.462261:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203892/real 1788203892] req@ffff93c1c5037100 x1875064559925248/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203892.462284:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559925248/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203892.462297:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203892.462301:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203892.462304:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559925248:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203892.462328:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559925376:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203892.462347:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203892.462357:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203892/real 1788203892] req@ffff93c1c5034a80 x1875064559925376/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203892.462365:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559925376/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203892.462372:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203892.462375:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203892.462377:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559925376:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203892.462381:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559925504:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203892.462400:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203892.462411:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203892/real 1788203892] req@ffff93c1c5034e00 x1875064559925504/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203892.462425:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559925504/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203892.462437:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203892.462439:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203892.462442:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559925504:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203893.166012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203894.190000:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203895.213997:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203896.237950:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203897.070000:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203897.070015:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203897.070111:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203897.070136:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203897.070140:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203897.070158:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203897.262024:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203897.582014:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203897.582077:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203897.582083:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203897.582094:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3028 00000100:00080000:2.0:1788203897.582107:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203897.582114:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203897.582160:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559925632/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203897.582185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203897.582188:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203897.582190:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203897.582194:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3028 00000100:00080000:2.0:1788203897.582196:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203897.582198:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203897.582208:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203897.582219:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559925760/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203897.582226:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203897.582229:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203897.582231:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203897.582234:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3028 00000100:00080000:2.0:1788203897.582236:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203897.582238:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203897.582242:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203897.582248:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559925632:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788203897.582255:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559925888/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203897.582337:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203897.582404:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203897/real 1788203897] req@ffff93c0e62bb800 x1875064559925632/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203897.582427:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559925632/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203897.582440:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203897.582445:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203897.582450:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559925632:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203897.582459:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559925760:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203897.582483:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203897.582496:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203897/real 1788203897] req@ffff93c0e62b9f80 x1875064559925760/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203897.582510:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559925760/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203897.582519:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203897.582523:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203897.582527:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559925760:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203897.582558:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559925888:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203897.582580:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203897.582592:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203897/real 1788203897] req@ffff93c0e62b8a80 x1875064559925888/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203897.582605:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559925888/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203897.582631:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203897.582636:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203897.582640:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559925888:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203898.286016:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203899.310049:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203900.333991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203901.358012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203902.190025:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203902.190047:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203902.191490:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203902.191524:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203902.191529:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203902.191562:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203902.381976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203902.701998:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203902.702010:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203902.702015:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203902.702024:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3033 00000100:00080000:3.0:1788203902.702034:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203902.702040:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203902.702071:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559926016/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203902.702092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203902.702095:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203902.702096:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203902.702099:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3033 00000100:00080000:3.0:1788203902.702101:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203902.702103:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203902.702111:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203902.702119:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559926144/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203902.702127:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203902.702130:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203902.702131:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203902.702134:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3033 00000100:00080000:3.0:1788203902.702136:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203902.702138:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203902.702143:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203902.702149:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559926272/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203902.702179:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559926016:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203902.702321:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559926144:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203902.702364:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559926272:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203903.406062:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203904.430056:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203905.454027:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788203905.774125:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788203905.774173:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788203905.774191:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788203905.774271:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788203905.774283:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788203905.774298:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203902/real 1788203905] req@ffff93c1c4a4d180 x1875064559926016/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203905.774328:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559926016/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203905.774339:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203905.774344:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203905.774348:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559926016:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203905.774357:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203902/real 1788203905] req@ffff93c1c4a4f800 x1875064559926144/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203905.774369:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559926144/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203905.774377:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203905.774381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203905.774385:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559926144:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203905.774391:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203902/real 1788203905] req@ffff93c1c4a4d880 x1875064559926272/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203905.774402:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559926272/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203905.774412:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203905.774416:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203905.774419:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559926272:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203906.477929:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203907.310095:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203907.310112:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203907.310213:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203907.310237:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203907.310240:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203907.310257:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203907.501994:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203907.822002:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203907.822014:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203907.822019:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203907.822031:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3038 00000100:00080000:1.0:1788203907.822043:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203907.822051:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203907.822090:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559926400/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203907.822117:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203907.822121:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203907.822123:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203907.822128:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3038 00000100:00080000:1.0:1788203907.822132:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203907.822137:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203907.822152:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203907.822162:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559926400:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203907.822170:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559926528/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203907.822186:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203907.822189:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203907.822191:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203907.822194:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3038 00000100:00080000:1.0:1788203907.822197:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203907.822201:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203907.822205:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203907.822220:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559926656/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203907.822257:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203907.822330:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203907/real 1788203907] req@ffff93c1c5036680 x1875064559926400/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203907.822355:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559926400/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203907.822369:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203907.822375:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203907.822381:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559926400:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203907.822420:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559926528:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203907.822448:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203907.822463:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203907/real 1788203907] req@ffff93c1c5035c00 x1875064559926528/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203907.822478:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559926528/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203907.822488:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203907.822492:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203907.822496:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559926528:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203907.822503:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559926656:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203907.822527:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203907.822541:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203907/real 1788203907] req@ffff93c1c5037800 x1875064559926656/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203907.822555:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559926656/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203907.822567:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203907.822570:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203907.822574:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559926656:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203908.525970:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203909.550049:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203910.573938:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203911.597938:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788203912.430134:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203912.430150:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203912.430251:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788203912.430270:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788203912.430273:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788203912.430290:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203912.621962:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203912.941984:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203912.941999:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788203912.942004:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203912.942018:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3043 00000100:00080000:2.0:1788203912.942033:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203912.942043:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788203912.942095:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559926784/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788203912.942140:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203912.942145:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788203912.942147:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203912.942152:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3043 00000100:00080000:2.0:1788203912.942157:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203912.942163:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203912.942176:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788203912.942193:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559926912/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203912.942209:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559926784:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788203912.942226:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203912.942232:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788203912.942234:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788203912.942242:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3043 00000100:00080000:2.0:1788203912.942248:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788203912.942253:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788203912.942260:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788203912.942276:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559927040/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203912.942301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203912.942355:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203912/real 1788203912] req@ffff93c0e62b9c00 x1875064559926784/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203912.942372:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559926784/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203912.942382:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203912.942387:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203912.942390:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559926784:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203912.942411:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559926912:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203912.942428:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203912.942437:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203912/real 1788203912] req@ffff93c0e62b9500 x1875064559926912/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203912.942445:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559926912/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203912.942451:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203912.942453:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203912.942455:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559926912:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203912.942459:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559927040:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203912.942471:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203912.942476:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203912/real 1788203912] req@ffff93c0e62bad80 x1875064559927040/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203912.942484:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559927040/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203912.942495:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203912.942497:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203912.942499:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559927040:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203913.645991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203914.669925:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203915.693988:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203916.718029:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203917.550040:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203917.550054:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203917.550148:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203917.550172:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203917.550176:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203917.550206:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203917.742021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203918.061986:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203918.061998:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203918.062003:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203918.062014:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3048 00000100:00080000:0.0:1788203918.062026:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203918.062033:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203918.062082:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559927168/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203918.062118:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203918.062122:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203918.062124:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203918.062128:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3048 00000100:00080000:0.0:1788203918.062132:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203918.062136:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203918.062147:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203918.062165:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559927296/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788203918.062165:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559927168:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203918.062182:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203918.062186:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203918.062188:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203918.062192:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3048 00000100:00080000:0.0:1788203918.062197:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203918.062201:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203918.062206:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203918.062220:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559927424/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788203918.062240:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203918.062287:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203918/real 1788203918] req@ffff93c0e9ce4380 x1875064559927168/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203918.062316:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559927168/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203918.062324:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203918.062327:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203918.062330:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559927168:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788203918.062349:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559927296:192.168.201.156@tcp:38:kworker.0 00000800:00000100:3.0:1788203918.062368:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203918.062377:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203918/real 1788203918] req@ffff93c0e9ce4700 x1875064559927296/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203918.062386:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559927296/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203918.062392:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203918.062394:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203918.062396:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559927296:192.168.201.156@tcp:38:kworker.0 00000100:00100000:3.0:1788203918.062400:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559927424:192.168.201.156@tcp:8:kworker.0 00000800:00000100:3.0:1788203918.062411:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788203918.062417:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203918/real 1788203918] req@ffff93c0e9ce7480 x1875064559927424/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788203918.062424:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559927424/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788203918.062431:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788203918.062432:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:3.0:1788203918.062434:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559927424:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203918.766010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203919.789947:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203920.814014:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203921.837997:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203922.670084:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203922.670105:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203922.670229:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203922.670255:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203922.670259:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203922.670283:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203922.862047:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203923.181995:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203923.182003:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203923.182007:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203923.182015:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3054 00000100:00080000:3.0:1788203923.182025:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203923.182031:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203923.182062:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ca80 x1875064559927552/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203923.182091:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203923.182094:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203923.182095:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203923.182098:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3054 00000100:00080000:3.0:1788203923.182100:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203923.182102:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203923.182110:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203923.182122:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d500 x1875064559927680/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203923.182133:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203923.182137:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203923.182140:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203923.182145:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3054 00000100:00100000:0.0:1788203923.182151:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559927552:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203923.182151:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203923.182160:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203923.182166:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203923.182180:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559927808/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203923.182219:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203923.182263:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203923/real 1788203923] req@ffff93c1c4a4ca80 x1875064559927552/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203923.182282:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ca80 x1875064559927552/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203923.182293:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203923.182298:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203923.182302:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559927552:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203923.182311:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559927680:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203923.182334:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203923.182353:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203923/real 1788203923] req@ffff93c1c4a4d500 x1875064559927680/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203923.182367:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d500 x1875064559927680/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203923.182376:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203923.182380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203923.182384:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559927680:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203923.182409:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559927808:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203923.182433:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203923.182446:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203923/real 1788203923] req@ffff93c1c4a4d880 x1875064559927808/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203923.182460:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559927808/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203923.182471:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203923.182474:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203923.182478:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559927808:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203923.885974:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203924.910041:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203925.933954:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203926.957988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203927.790048:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203927.790069:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203927.790200:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203927.790236:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203927.790242:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203927.790276:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203927.981982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203928.301972:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203928.301988:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203928.301994:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203928.302006:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3059 00000100:00080000:3.0:1788203928.302021:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203928.302031:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203928.302090:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559927936/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203928.302117:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203928.302120:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203928.302121:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203928.302125:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3059 00000100:00080000:3.0:1788203928.302128:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203928.302131:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203928.302141:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203928.302152:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559928064/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203928.302166:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559927936:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203928.302167:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203928.302176:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203928.302178:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203928.302182:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3059 00000100:00080000:3.0:1788203928.302184:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203928.302186:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203928.302190:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203928.302198:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559928192/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203928.302254:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203928.302311:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203928/real 1788203928] req@ffff93c1c4a4f800 x1875064559927936/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203928.302327:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559927936/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203928.302336:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203928.302341:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203928.302346:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559927936:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203928.302373:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559928064:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203928.302392:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203928.302402:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203928/real 1788203928] req@ffff93c1c4a4d180 x1875064559928064/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203928.302411:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559928064/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203928.302417:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203928.302419:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203928.302422:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559928064:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203928.302425:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559928192:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203928.302437:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203928.302471:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203928/real 1788203928] req@ffff93c1c4a4e300 x1875064559928192/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203928.302481:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559928192/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203928.302490:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203928.302492:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203928.302494:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559928192:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203929.006048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203930.029944:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203931.054033:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203932.077956:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203932.910049:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203932.910073:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203932.910210:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203932.910249:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203932.910255:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203932.910287:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203933.102060:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203933.422005:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203933.422016:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203933.422021:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203933.422032:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3064 00000100:00080000:1.0:1788203933.422044:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203933.422052:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203933.422101:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559928320/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203933.422127:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203933.422130:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203933.422132:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203933.422135:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3064 00000100:00080000:1.0:1788203933.422137:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203933.422140:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203933.422152:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203933.422164:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559928448/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788203933.422174:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559928320:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788203933.422181:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203933.422187:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203933.422189:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203933.422195:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3064 00000100:00080000:1.0:1788203933.422200:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203933.422205:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203933.422212:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203933.422227:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559928576/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203933.422250:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203933.422293:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203933/real 1788203933] req@ffff93c1c5037800 x1875064559928320/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203933.422308:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559928320/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203933.422317:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203933.422320:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203933.422324:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559928320:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203933.422348:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559928448:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203933.422384:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203933.422395:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203933/real 1788203933] req@ffff93c1c5035c00 x1875064559928448/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203933.422409:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559928448/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203933.422419:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203933.422422:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203933.422426:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559928448:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203933.422433:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559928576:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203933.422457:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203933.422471:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203933/real 1788203933] req@ffff93c1c5036680 x1875064559928576/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203933.422485:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559928576/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203933.422497:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203933.422501:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203933.422505:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559928576:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203934.126017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203935.149986:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203936.173956:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203937.197967:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203938.029994:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203938.030010:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203938.030115:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203938.030137:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203938.030140:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203938.030159:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203938.221998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203938.541906:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203938.541914:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203938.541918:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203938.541926:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3069 00000100:00080000:1.0:1788203938.541935:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203938.541941:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203938.541974:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559928704/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203938.541998:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203938.542001:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203938.542003:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203938.542006:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3069 00000100:00080000:1.0:1788203938.542008:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203938.542010:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203938.542018:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203938.542029:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559928832/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203938.542039:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203938.542043:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203938.542045:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203938.542052:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3069 00000100:00100000:0.0:1788203938.542053:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559928704:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788203938.542056:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203938.542064:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203938.542070:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203938.542084:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559928960/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203938.542118:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203938.542164:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203938/real 1788203938] req@ffff93c1c5034e00 x1875064559928704/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203938.542180:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559928704/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203938.542190:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203938.542193:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203938.542197:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559928704:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203938.542202:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559928832:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203938.542217:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203938.542225:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203938/real 1788203938] req@ffff93c1c5034a80 x1875064559928832/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203938.542233:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559928832/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203938.542238:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203938.542240:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203938.542242:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559928832:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203938.542260:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559928960:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203938.542300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203938.542308:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203938/real 1788203938] req@ffff93c1c5037100 x1875064559928960/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203938.542316:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559928960/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203938.542327:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203938.542330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203938.542332:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559928960:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203939.245991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203940.269987:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203941.294015:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203942.318011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203943.149985:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203943.149999:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203943.150096:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203943.150120:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203943.150123:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203943.150140:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203943.342029:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203943.661971:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203943.661983:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203943.661987:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203943.661999:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3074 00000100:00080000:1.0:1788203943.662010:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203943.662019:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203943.662056:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559929088/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203943.662077:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203943.662080:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203943.662081:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203943.662084:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3074 00000100:00080000:1.0:1788203943.662088:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203943.662092:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203943.662100:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203943.662107:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559929088:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203943.662111:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559929216/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203943.662122:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203943.662125:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203943.662126:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203943.662129:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3074 00000100:00080000:1.0:1788203943.662132:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203943.662134:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203943.662138:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203943.662147:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559929344/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203943.662192:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203943.662234:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203943/real 1788203943] req@ffff93c1c5037100 x1875064559929088/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203943.662250:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559929088/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203943.662258:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203943.662263:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203943.662267:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559929088:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203943.662292:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559929216:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203943.662318:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203943.662329:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203943/real 1788203943] req@ffff93c1c5034a80 x1875064559929216/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203943.662338:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559929216/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203943.662344:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203943.662347:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203943.662349:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559929216:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203943.662353:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559929344:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203943.662367:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203943.662373:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203943/real 1788203943] req@ffff93c1c5034e00 x1875064559929344/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203943.662381:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559929344/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203943.662387:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203943.662389:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203943.662391:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559929344:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203944.365993:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203945.390008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203946.414056:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203947.438031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203948.270079:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203948.270117:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203948.270230:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203948.270256:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203948.270261:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203948.270292:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203948.462025:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203948.782000:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203948.782023:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203948.782029:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203948.782043:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3079 00000100:00080000:0.0:1788203948.782056:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203948.782067:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203948.782131:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559929472/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203948.782160:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203948.782163:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203948.782164:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203948.782168:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3079 00000100:00080000:0.0:1788203948.782171:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203948.782176:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203948.782186:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788203948.782198:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559929472:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203948.782200:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559929600/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203948.782214:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203948.782217:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203948.782218:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203948.782222:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3079 00000100:00080000:0.0:1788203948.782224:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203948.782227:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203948.782230:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203948.782240:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559929728/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203948.782277:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203948.782329:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203948/real 1788203948] req@ffff93c0e9ce7480 x1875064559929472/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203948.782345:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559929472/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203948.782355:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203948.782358:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203948.782362:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559929472:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203948.782388:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559929600:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788203948.782404:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203948.782413:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203948/real 1788203948] req@ffff93c0e9ce4700 x1875064559929600/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203948.782421:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559929600/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203948.782427:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203948.782430:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203948.782433:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559929600:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788203948.782437:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559929728:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788203948.782450:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203948.782456:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203948/real 1788203948] req@ffff93c0e9ce4380 x1875064559929728/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203948.782464:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559929728/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203948.782472:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203948.782474:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203948.782475:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559929728:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203949.485978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203950.510038:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203951.533992:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203952.558006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203953.390042:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203953.390059:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203953.390173:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203953.390204:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203953.390210:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203953.390237:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203953.581975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203953.901932:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203953.901944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203953.901948:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203953.901959:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3084 00000100:00080000:3.0:1788203953.901972:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203953.901981:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203953.902043:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559929856/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203953.902069:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203953.902072:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203953.902074:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203953.902078:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3084 00000100:00080000:3.0:1788203953.902081:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203953.902084:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203953.902094:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203953.902104:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559929984/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203953.902113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203953.902116:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203953.902117:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203953.902123:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3084 00000100:00100000:1.0:1788203953.902124:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559929856:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203953.902125:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203953.902132:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203953.902137:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203953.902146:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559930112/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203953.902208:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203953.902257:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203953/real 1788203953] req@ffff93c1c4a4c700 x1875064559929856/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203953.902273:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559929856/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203953.902282:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203953.902285:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203953.902290:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559929856:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203953.902322:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559929984:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788203953.902339:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203953.902352:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203953/real 1788203953] req@ffff93c1c4a4c380 x1875064559929984/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203953.902366:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559929984/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203953.902376:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203953.902379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203953.902383:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559929984:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788203953.902390:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559930112:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788203953.902411:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203953.902424:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203953/real 1788203953] req@ffff93c1c4a4f480 x1875064559930112/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203953.902433:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559930112/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203953.902442:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203953.902444:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203953.902446:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559930112:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203954.606021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203955.630058:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203956.653989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203957.678056:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203958.510075:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203958.510093:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203958.510209:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203958.510237:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203958.510242:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203958.510275:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203958.701983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203959.021954:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203959.021965:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203959.021969:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203959.021980:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3089 00000100:00080000:0.0:1788203959.021991:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203959.021999:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203959.022038:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559930240/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203959.022059:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203959.022066:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203959.022067:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203959.022071:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3089 00000100:00080000:0.0:1788203959.022074:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203959.022077:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203959.022086:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203959.022096:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559930368/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203959.022101:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559930240:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203959.022111:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203959.022114:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203959.022115:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203959.022118:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3089 00000100:00080000:0.0:1788203959.022120:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203959.022122:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203959.022126:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203959.022136:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559930496/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203959.022175:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203959.022233:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203959/real 1788203959] req@ffff93c0e9ce6a00 x1875064559930240/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203959.022255:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559930240/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203959.022269:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203959.022274:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203959.022277:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559930240:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203959.022298:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559930368:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788203959.022318:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203959.022328:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203959/real 1788203959] req@ffff93c0e9ce4000 x1875064559930368/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203959.022337:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559930368/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203959.022343:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203959.022345:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203959.022348:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559930368:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788203959.022352:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559930496:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788203959.022364:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203959.022370:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203959/real 1788203959] req@ffff93c0e9ce6680 x1875064559930496/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203959.022378:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559930496/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203959.022385:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203959.022387:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203959.022389:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559930496:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203959.726042:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203960.749979:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203961.774090:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203962.798004:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203963.630086:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203963.630105:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203963.630217:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203963.630244:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203963.630247:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203963.630272:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203963.821982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788203964.141938:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203964.141953:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788203964.141959:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203964.141972:0:18364:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3095 00000100:00080000:1.0:1788203964.141987:0:18364:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203964.141996:0:18364:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203964.142047:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559930624/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203964.142082:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203964.142086:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788203964.142088:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203964.142093:0:18364:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3095 00000100:00080000:1.0:1788203964.142098:0:18364:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203964.142103:0:18364:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203964.142114:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203964.142124:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559930624:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203964.142130:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559930752/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788203964.142143:0:18364:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788203964.142149:0:18364:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788203964.142152:0:18364:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788203964.142157:0:18364:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3095 00000100:00080000:1.0:1788203964.142161:0:18364:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788203964.142165:0:18364:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788203964.142171:0:18364:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788203964.142184:0:18364:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559930880/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788203964.142212:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203964.142279:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203964/real 1788203964] req@ffff93c1c5036680 x1875064559930624/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203964.142304:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559930624/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203964.142317:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203964.142322:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203964.142327:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559930624:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788203964.142356:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559930752:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788203964.142381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203964.142395:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203964/real 1788203964] req@ffff93c1c5035c00 x1875064559930752/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203964.142410:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559930752/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203964.142419:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203964.142423:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203964.142427:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559930752:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788203964.142433:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559930880:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788203964.142453:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788203964.142464:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203964/real 1788203964] req@ffff93c1c5037800 x1875064559930880/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203964.142477:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559930880/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203964.142496:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203964.142500:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788203964.142504:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559930880:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203964.845989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203965.869960:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203966.893952:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203967.918044:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203968.750002:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203968.750034:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203968.750194:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203968.750233:0:18364:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203968.750247:0:18364:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203968.750542:0:18364:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203968.941952:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203969.261958:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203969.261972:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203969.261977:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203969.261990:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3100 00000100:00080000:0.0:1788203969.262004:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203969.262012:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203969.262061:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559931008/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203969.262087:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203969.262090:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203969.262092:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203969.262096:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3100 00000100:00080000:0.0:1788203969.262099:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203969.262103:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203969.262113:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203969.262124:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559931136/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203969.262138:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788203969.262139:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559931008:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203969.262149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203969.262153:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203969.262160:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3100 00000100:00080000:0.0:1788203969.262174:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203969.262180:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203969.262187:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203969.262202:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559931264/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203969.262263:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559931136:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788203969.262296:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559931264:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203969.965965:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203970.990021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203972.013983:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788203972.334228:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:2.0:1788203972.334277:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788203972.334296:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:2.0:1788203972.334350:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:2.0:1788203972.334363:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:1.0:1788203972.334449:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203969/real 1788203972] req@ffff93c0e9ce5c00 x1875064559931136/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203972.334492:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559931136/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203972.334505:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203972.334510:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203972.334514:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559931136:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788203972.334531:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203969/real 1788203972] req@ffff93c0e9ce6d80 x1875064559931264/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203972.334546:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559931264/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203972.334558:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203972.334562:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203972.334566:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559931264:192.168.201.156@tcp:8:kworker.0 00000100:00100000:1.0:1788203972.334574:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203969/real 1788203972] req@ffff93c0e9ce7b80 x1875064559931008/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203972.334588:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559931008/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203972.334605:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203972.334609:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203972.334612:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559931008:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788203973.038034:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203973.869975:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203973.869988:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203973.870088:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203973.870109:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203973.870112:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203973.870131:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203974.062010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203974.381978:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203974.382002:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203974.382007:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203974.382021:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3105 00000100:00080000:3.0:1788203974.382035:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203974.382044:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203974.382091:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559931392/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203974.382126:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203974.382131:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203974.382133:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203974.382139:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3105 00000100:00080000:3.0:1788203974.382144:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203974.382149:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203974.382161:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203974.382176:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559931520/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203974.382189:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203974.382194:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203974.382197:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788203974.382202:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559931392:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203974.382205:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3105 00000100:00080000:3.0:1788203974.382212:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203974.382219:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203974.382224:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203974.382237:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559931648/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203974.382303:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203974.382367:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203974/real 1788203974] req@ffff93c1c4a4ea00 x1875064559931392/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203974.382393:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559931392/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203974.382407:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203974.382412:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203974.382417:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559931392:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203974.382454:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559931520:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788203974.382486:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203974.382502:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203974/real 1788203974] req@ffff93c1c4a4c000 x1875064559931520/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203974.382516:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559931520/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203974.382526:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203974.382530:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203974.382533:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559931520:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788203974.382540:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559931648:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788203974.382565:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203974.382577:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203974/real 1788203974] req@ffff93c1c4a4fb80 x1875064559931648/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203974.382588:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559931648/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203974.382600:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203974.382603:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203974.382607:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559931648:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203975.085992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203976.109952:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203977.133996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203978.157956:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788203978.990065:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203978.990083:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203978.990211:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788203978.990240:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788203978.990243:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788203978.990266:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203979.182004:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203979.501972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203979.501983:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203979.501987:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203979.501998:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3110 00000100:00080000:0.0:1788203979.502010:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203979.502018:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203979.502064:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559931776/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203979.502091:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203979.502094:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203979.502096:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203979.502100:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3110 00000100:00080000:0.0:1788203979.502103:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203979.502106:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203979.502119:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203979.502134:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559931904/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203979.502140:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559931776:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203979.502155:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203979.502160:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203979.502162:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203979.502169:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3110 00000100:00080000:0.0:1788203979.502174:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203979.502179:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203979.502185:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203979.502200:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559932032/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203979.502226:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203979.502288:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203979/real 1788203979] req@ffff93c0e9ce5f80 x1875064559931776/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203979.502310:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559931776/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203979.502318:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203979.502321:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203979.502332:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559931776:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203979.502359:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559931904:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788203979.502379:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203979.502388:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203979/real 1788203979] req@ffff93c0e9ce4a80 x1875064559931904/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203979.502397:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559931904/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203979.502403:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203979.502406:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203979.502408:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559931904:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788203979.502412:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559932032:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788203979.502423:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203979.502430:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203979/real 1788203979] req@ffff93c0e9ce7100 x1875064559932032/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203979.502446:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559932032/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203979.502454:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203979.502456:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203979.502458:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559932032:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203980.206074:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203981.230019:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203982.254020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203983.277968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203984.110021:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203984.110035:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203984.110142:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203984.110162:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203984.110166:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203984.110187:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203984.302003:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203984.621924:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203984.621933:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203984.621936:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203984.621945:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3115 00000100:00080000:0.0:1788203984.621954:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203984.621960:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203984.621993:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559932160/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203984.622016:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203984.622019:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203984.622020:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203984.622029:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3115 00000100:00080000:0.0:1788203984.622032:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203984.622034:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203984.622042:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203984.622051:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559932288/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203984.622069:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788203984.622072:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559932160:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203984.622076:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203984.622080:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203984.622086:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3115 00000100:00080000:0.0:1788203984.622090:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203984.622094:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203984.622101:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203984.622115:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559932416/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203984.622136:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203984.622183:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203984/real 1788203984] req@ffff93c0e9ce5180 x1875064559932160/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203984.622203:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559932160/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203984.622214:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203984.622219:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203984.622224:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559932160:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203984.622247:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559932288:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788203984.622266:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203984.622276:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203984/real 1788203984] req@ffff93c0e9ce5500 x1875064559932288/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203984.622284:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559932288/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203984.622294:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203984.622297:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203984.622299:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559932288:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788203984.622303:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559932416:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788203984.622314:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203984.622320:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203984/real 1788203984] req@ffff93c0e9ce7800 x1875064559932416/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203984.622327:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559932416/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203984.622334:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203984.622336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203984.622338:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559932416:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203985.326001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203986.349937:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203987.373986:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203988.397980:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788203989.230055:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203989.230069:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203989.230159:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788203989.230178:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788203989.230181:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788203989.230198:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203989.421979:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788203989.741992:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203989.742003:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788203989.742007:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203989.742018:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3120 00000100:00080000:3.0:1788203989.742037:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203989.742045:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788203989.742084:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559932544/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788203989.742116:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203989.742121:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788203989.742123:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203989.742128:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3120 00000100:00080000:3.0:1788203989.742132:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203989.742137:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203989.742147:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788203989.742162:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559932672/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788203989.742167:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559932544:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788203989.742184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788203989.742187:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788203989.742190:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788203989.742195:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3120 00000100:00080000:3.0:1788203989.742199:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788203989.742203:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788203989.742209:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788203989.742225:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559932800/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203989.742234:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203989.742285:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203989/real 1788203989] req@ffff93c1c4a4ed80 x1875064559932544/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203989.742305:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559932544/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203989.742314:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203989.742317:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203989.742320:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559932544:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203989.742339:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559932672:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788203989.742357:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203989.742369:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203989/real 1788203989] req@ffff93c1c4a4dc00 x1875064559932672/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203989.742395:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559932672/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203989.742405:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203989.742408:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203989.742412:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559932672:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788203989.742419:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559932800:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788203989.742440:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203989.742449:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203989/real 1788203989] req@ffff93c1c4a4e680 x1875064559932800/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203989.742457:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559932800/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203989.742464:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203989.742466:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203989.742468:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559932800:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203990.445957:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203991.469981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203992.494005:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203993.517966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203994.349981:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203994.349995:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203994.350101:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203994.350120:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203994.350124:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203994.350144:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203994.541991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203994.861925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203994.861932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203994.861936:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203994.861944:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3125 00000100:00080000:0.0:1788203994.861954:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203994.861960:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203994.861992:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559932928/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203994.862013:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203994.862016:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203994.862017:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203994.862020:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3125 00000100:00080000:0.0:1788203994.862022:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203994.862024:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203994.862032:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203994.862041:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559933056/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203994.862049:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203994.862052:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203994.862053:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203994.862055:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3125 00000100:00080000:0.0:1788203994.862057:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203994.862062:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788203994.862064:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559932928:192.168.201.156@tcp:250:kworker.0 00000008:00100000:0.0:1788203994.862066:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203994.862081:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559933184/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203994.862124:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203994.862168:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203994/real 1788203994] req@ffff93c0e9ce4e00 x1875064559932928/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203994.862188:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559932928/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203994.862199:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203994.862204:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203994.862208:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559932928:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203994.862221:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559933056:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788203994.862239:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203994.862249:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203994/real 1788203994] req@ffff93c0e9ce5880 x1875064559933056/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203994.862257:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559933056/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203994.862263:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203994.862265:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203994.862267:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559933056:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788203994.862315:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559933184:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788203994.862334:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203994.862347:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203994/real 1788203994] req@ffff93c0e9ce6300 x1875064559933184/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203994.862360:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559933184/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203994.862378:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203994.862382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203994.862385:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559933184:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788203995.565992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203996.589974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203997.613984:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788203998.639461:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788203999.470103:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203999.470130:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203999.470304:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788203999.470342:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788203999.470347:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788203999.470372:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788203999.661994:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203999.981968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203999.981982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788203999.981986:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203999.981996:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3130 00000100:00080000:0.0:1788203999.982007:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203999.982014:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788203999.982056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559933312/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203999.982080:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203999.982083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788203999.982085:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203999.982088:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3130 00000100:00080000:0.0:1788203999.982090:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203999.982092:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203999.982103:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788203999.982113:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559933440/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788203999.982121:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203999.982123:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788203999.982125:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788203999.982130:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3130 00000100:00100000:1.0:1788203999.982130:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559933312:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788203999.982133:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788203999.982138:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788203999.982142:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788203999.982148:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559933568/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788203999.982229:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203999.982288:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203999/real 1788203999] req@ffff93c0e9ce6300 x1875064559933312/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203999.982304:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559933312/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203999.982312:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203999.982316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203999.982319:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559933312:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788203999.982342:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559933440:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788203999.982364:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203999.982377:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203999/real 1788203999] req@ffff93c0e9ce5880 x1875064559933440/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203999.982390:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559933440/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203999.982398:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203999.982401:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203999.982403:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559933440:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788203999.982408:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559933568:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788203999.982424:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788203999.982431:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203999/real 1788203999] req@ffff93c0e9ce4e00 x1875064559933568/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788203999.982438:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559933568/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788203999.982445:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788203999.982447:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788203999.982449:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559933568:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204000.685982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204001.709944:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204002.734016:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204003.757956:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204004.589977:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204004.589989:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204004.590081:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204004.590097:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204004.590099:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204004.590135:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204004.782014:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204005.101935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204005.101947:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204005.101952:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204005.101964:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3135 00000100:00080000:1.0:1788204005.101977:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204005.101985:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204005.102037:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559933696/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204005.102065:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204005.102068:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204005.102070:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204005.102075:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3135 00000100:00080000:1.0:1788204005.102078:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204005.102084:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204005.102095:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204005.102096:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559933696:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204005.102120:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559933824/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204005.102132:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204005.102135:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204005.102137:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204005.102142:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3135 00000100:00080000:1.0:1788204005.102145:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204005.102149:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204005.102154:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204005.102164:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559933952/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204005.102177:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204005.102219:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204005/real 1788204005] req@ffff93c1c5037800 x1875064559933696/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204005.102233:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559933696/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204005.102242:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204005.102245:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204005.102250:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559933696:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204005.102276:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559933824:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204005.102299:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204005.102309:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204005/real 1788204005] req@ffff93c1c5035c00 x1875064559933824/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204005.102318:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559933824/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204005.102323:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204005.102325:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204005.102327:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559933824:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204005.102332:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559933952:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204005.102343:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204005.102349:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204005/real 1788204005] req@ffff93c1c5036680 x1875064559933952/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204005.102357:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559933952/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204005.102368:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204005.102370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204005.102372:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559933952:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204005.806001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204006.830041:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204007.853972:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204008.877986:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204009.710012:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204009.710048:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204009.710144:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204009.710166:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204009.710169:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204009.710183:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204009.902044:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204010.221927:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204010.221938:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204010.221942:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204010.221952:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3141 00000100:00080000:3.0:1788204010.221964:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204010.221971:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204010.222013:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4df80 x1875064559934080/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204010.222050:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204010.222054:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204010.222056:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204010.222060:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3141 00000100:00080000:3.0:1788204010.222064:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204010.222068:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204010.222080:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204010.222093:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f100 x1875064559934208/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204010.222102:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204010.222104:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204010.222107:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204010.222110:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559934080:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788204010.222112:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3141 00000100:00080000:3.0:1788204010.222117:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204010.222121:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204010.222125:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788204010.222134:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559934336/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204010.222194:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204010.222247:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204010/real 1788204010] req@ffff93c1c4a4df80 x1875064559934080/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204010.222268:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4df80 x1875064559934080/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204010.222279:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204010.222284:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204010.222288:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559934080:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204010.222316:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559934208:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204010.222339:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204010.222364:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204010/real 1788204010] req@ffff93c1c4a4f100 x1875064559934208/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204010.222376:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f100 x1875064559934208/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204010.222386:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204010.222390:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204010.222393:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559934208:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204010.222399:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559934336:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204010.222415:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204010.222423:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204010/real 1788204010] req@ffff93c1c4a4e680 x1875064559934336/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204010.222434:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559934336/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204010.222443:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204010.222446:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204010.222449:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559934336:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204010.926065:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204011.950012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204012.973964:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204013.998004:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204014.830059:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204014.830079:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204014.830213:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204014.830242:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204014.830247:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204014.830287:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204015.021974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204015.342178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204015.342192:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204015.342197:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204015.342210:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3146 00000100:00080000:1.0:1788204015.342223:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204015.342231:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204015.342270:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559934464/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204015.342299:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204015.342302:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204015.342305:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204015.342308:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3146 00000100:00080000:1.0:1788204015.342311:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204015.342314:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204015.342324:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204015.342334:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559934592/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204015.342349:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204015.342356:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204015.342359:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204015.342361:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559934464:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204015.342366:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3146 00000100:00080000:1.0:1788204015.342371:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204015.342375:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204015.342381:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204015.342395:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559934720/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204015.342442:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204015.342488:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204015/real 1788204015] req@ffff93c1c5034e00 x1875064559934464/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204015.342508:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559934464/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204015.342520:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204015.342525:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204015.342530:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559934464:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204015.342559:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559934592:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204015.342590:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204015.342607:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204015/real 1788204015] req@ffff93c1c5034a80 x1875064559934592/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204015.342621:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559934592/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204015.342630:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204015.342634:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204015.342638:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559934592:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204015.342645:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559934720:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204015.342667:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204015.342679:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204015/real 1788204015] req@ffff93c1c5037100 x1875064559934720/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204015.342698:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559934720/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204015.342709:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204015.342713:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204015.342716:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559934720:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204016.046017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204017.069996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204018.094012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204019.117986:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204019.950043:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204019.950058:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204019.950170:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204019.950198:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204019.950202:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204019.950230:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204020.141993:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204020.461937:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204020.461944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204020.461947:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204020.461955:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3151 00000100:00080000:3.0:1788204020.461963:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204020.461968:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204020.461997:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559934848/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204020.462016:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204020.462018:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204020.462019:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204020.462022:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3151 00000100:00080000:3.0:1788204020.462025:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204020.462027:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204020.462035:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204020.462042:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559934976/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204020.462048:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204020.462050:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204020.462051:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204020.462054:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3151 00000100:00080000:3.0:1788204020.462056:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204020.462058:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204020.462061:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204020.462067:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559934848:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788204020.462068:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559935104/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204020.462132:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204020.462178:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204020/real 1788204020] req@ffff93c1c4a4dc00 x1875064559934848/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204020.462193:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559934848/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204020.462200:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204020.462204:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204020.462207:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559934848:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204020.462213:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559934976:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204020.462228:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204020.462235:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204020/real 1788204020] req@ffff93c1c4a4ed80 x1875064559934976/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204020.462242:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559934976/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204020.462247:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204020.462250:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204020.462253:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559934976:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204020.462276:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559935104:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204020.462296:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204020.462307:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204020/real 1788204020] req@ffff93c1c4a4fb80 x1875064559935104/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204020.462319:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559935104/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204020.462329:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204020.462332:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204020.462335:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559935104:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204021.165979:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204022.189957:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204023.213972:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204024.238020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204025.070023:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204025.070036:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204025.070128:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204025.070148:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204025.070151:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204025.070166:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204025.261959:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204025.581982:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204025.581993:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204025.581997:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204025.582007:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3156 00000100:00080000:1.0:1788204025.582018:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204025.582025:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204025.582061:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559935232/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204025.582088:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204025.582095:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204025.582097:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204025.582102:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3156 00000100:00080000:1.0:1788204025.582105:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204025.582108:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204025.582118:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204025.582130:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559935360/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204025.582134:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559935232:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204025.582148:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204025.582151:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204025.582153:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204025.582158:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3156 00000100:00080000:1.0:1788204025.582161:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204025.582165:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204025.582170:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204025.582181:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559935488/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204025.582193:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204025.582234:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204025/real 1788204025] req@ffff93c1c5037100 x1875064559935232/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204025.582264:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559935232/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204025.582274:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204025.582278:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204025.582281:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559935232:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204025.582304:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559935360:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204025.582326:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204025.582335:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204025/real 1788204025] req@ffff93c1c5034a80 x1875064559935360/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204025.582343:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559935360/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204025.582357:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204025.582360:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204025.582363:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559935360:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204025.582369:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559935488:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204025.582389:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204025.582400:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204025/real 1788204025] req@ffff93c1c5034e00 x1875064559935488/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204025.582409:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559935488/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204025.582419:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204025.582421:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204025.582423:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559935488:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204026.286017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204027.309962:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204028.334056:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204029.357923:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204030.190018:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204030.190031:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204030.190117:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204030.190136:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204030.190138:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204030.190154:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204030.382029:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204030.701978:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204030.701988:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204030.701991:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204030.702001:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3161 00000100:00080000:3.0:1788204030.702012:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204030.702019:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204030.702053:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559935616/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204030.702072:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204030.702074:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204030.702075:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204030.702078:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3161 00000100:00080000:3.0:1788204030.702080:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204030.702083:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204030.702091:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204030.702099:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559935744/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204030.702106:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204030.702109:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788204030.702112:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559935616:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788204030.702112:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204030.702116:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3161 00000100:00080000:3.0:1788204030.702119:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204030.702122:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204030.702125:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788204030.702132:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559935872/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204030.702168:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204030.702204:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204030/real 1788204030] req@ffff93c1c4a4c000 x1875064559935616/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204030.702216:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559935616/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204030.702224:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204030.702227:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204030.702229:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559935616:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204030.702250:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559935744:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204030.702264:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204030.702271:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204030/real 1788204030] req@ffff93c1c4a4ea00 x1875064559935744/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204030.702278:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559935744/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204030.702283:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204030.702285:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204030.702287:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559935744:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204030.702291:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559935872:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204030.702300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204030.702306:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204030/real 1788204030] req@ffff93c1c4a4f480 x1875064559935872/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204030.702312:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559935872/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204030.702318:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204030.702319:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204030.702321:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559935872:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204031.405978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204032.430012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204033.453970:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204034.478032:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204035.309977:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204035.309993:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204035.310111:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204035.310138:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204035.310150:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204035.310175:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204035.501990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204035.821985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204035.821994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204035.821997:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204035.822006:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3166 00000100:00080000:1.0:1788204035.822016:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204035.822022:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204035.822057:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559936000/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204035.822089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204035.822093:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204035.822094:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204035.822098:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3166 00000100:00080000:1.0:1788204035.822102:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204035.822105:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204035.822114:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204035.822126:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559936128/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204035.822133:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559936000:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204035.822145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204035.822147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204035.822148:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204035.822151:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3166 00000100:00080000:1.0:1788204035.822153:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204035.822156:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204035.822160:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204035.822167:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559936256/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204035.822227:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559936128:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204035.822248:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559936256:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204036.525981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204037.550020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204038.574028:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788204038.894100:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788204038.894145:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788204038.894163:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788204038.894206:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788204038.894217:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788204038.894283:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204035/real 1788204038] req@ffff93c1c5035c00 x1875064559936128/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204038.894319:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559936128/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204038.894340:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204038.894347:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204038.894352:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559936128:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204038.894365:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204035/real 1788204038] req@ffff93c1c5037800 x1875064559936256/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204038.894379:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559936256/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204038.894392:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204038.894396:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204038.894400:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559936256:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788204038.894411:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204035/real 1788204038] req@ffff93c1c5036680 x1875064559936000/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204038.894423:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559936000/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204038.894430:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204038.894434:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204038.894437:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559936000:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788204039.597975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204040.429963:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204040.429982:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204040.430078:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204040.430105:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204040.430108:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204040.430124:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204040.622028:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204040.942058:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204040.942073:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204040.942079:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204040.942093:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3171 00000100:00080000:1.0:1788204040.942107:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204040.942117:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204040.942182:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559936384/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204040.942218:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204040.942222:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204040.942224:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204040.942228:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3171 00000100:00080000:1.0:1788204040.942232:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204040.942237:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204040.942248:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204040.942263:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559936512/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204040.942272:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204040.942276:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204040.942277:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204040.942282:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3171 00000100:00080000:1.0:1788204040.942284:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204040.942287:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204040.942292:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204040.942300:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559936640/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204040.942767:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559936384:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788204040.942891:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204040.942962:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204040/real 1788204040] req@ffff93c1c5036680 x1875064559936384/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204040.942985:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559936384/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204040.942998:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204040.943003:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204040.943007:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559936384:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204040.943016:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559936512:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204040.943042:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204040.943057:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204040/real 1788204040] req@ffff93c1c5037800 x1875064559936512/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204040.943068:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559936512/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204040.943075:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204040.943079:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204040.943081:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559936512:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204040.943088:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559936640:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204040.943107:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204040.943116:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204040/real 1788204040] req@ffff93c1c5035c00 x1875064559936640/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204040.943126:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559936640/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204040.943142:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204040.943146:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204040.943148:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559936640:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204041.646006:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204042.670010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204043.693993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204044.718031:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204045.550021:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204045.550041:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204045.550149:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204045.550171:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204045.550174:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204045.550191:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204045.742046:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204046.062007:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204046.062022:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204046.062027:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204046.062039:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3176 00000100:00080000:0.0:1788204046.062050:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204046.062057:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204046.062101:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559936768/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204046.062128:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204046.062130:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204046.062132:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204046.062134:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3176 00000100:00080000:0.0:1788204046.062137:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204046.062140:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204046.062149:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204046.062158:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559936896/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204046.062165:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204046.062168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204046.062169:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204046.062174:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3176 00000100:00100000:1.0:1788204046.062175:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559936768:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204046.062176:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204046.062181:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204046.062188:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204046.062194:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559937024/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204046.062260:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204046.062318:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204046/real 1788204046] req@ffff93c0e9ce7800 x1875064559936768/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204046.062341:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559936768/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204046.062353:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204046.062357:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204046.062362:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559936768:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204046.062394:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559936896:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204046.062427:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204046.062440:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204046/real 1788204046] req@ffff93c0e9ce5500 x1875064559936896/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204046.062453:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559936896/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204046.062461:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204046.062464:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204046.062468:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559936896:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204046.062474:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559937024:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204046.062493:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204046.062505:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204046/real 1788204046] req@ffff93c0e9ce5180 x1875064559937024/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204046.062523:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559937024/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204046.062535:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204046.062538:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204046.062541:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559937024:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204046.765979:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204047.790055:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204048.814097:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204049.838008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204050.670020:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204050.670039:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204050.670190:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204050.670223:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204050.670228:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204050.670258:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204050.862038:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204051.181937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204051.181952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204051.181958:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204051.181970:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3182 00000100:00080000:1.0:1788204051.181992:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204051.182001:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204051.182050:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559937152/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204051.182084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204051.182088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204051.182090:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204051.182094:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3182 00000100:00080000:1.0:1788204051.182099:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204051.182103:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204051.182114:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204051.182131:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559937280/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204051.182132:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559937152:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204051.182148:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204051.182152:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204051.182154:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204051.182159:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3182 00000100:00080000:1.0:1788204051.182163:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204051.182166:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204051.182171:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204051.182184:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559937408/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204051.182214:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204051.182252:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204051/real 1788204051] req@ffff93c1c5034e00 x1875064559937152/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204051.182266:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559937152/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204051.182275:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204051.182278:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204051.182281:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559937152:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204051.182303:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559937280:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204051.182341:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204051.182353:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204051/real 1788204051] req@ffff93c1c5034a80 x1875064559937280/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204051.182366:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559937280/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204051.182374:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204051.182377:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204051.182381:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559937280:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204051.182387:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559937408:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204051.182408:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204051.182419:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204051/real 1788204051] req@ffff93c1c5037100 x1875064559937408/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204051.182431:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559937408/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204051.182441:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204051.182444:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204051.182447:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559937408:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204051.885934:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204052.910017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204053.934012:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204054.958059:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204055.790018:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204055.790035:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204055.790128:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204055.790153:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204055.790157:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204055.790172:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204055.981991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204056.301963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204056.301974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204056.301977:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204056.301999:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3187 00000100:00080000:1.0:1788204056.302012:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204056.302020:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204056.302060:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559937536/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204056.302086:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204056.302090:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204056.302092:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204056.302096:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3187 00000100:00080000:1.0:1788204056.302099:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204056.302103:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204056.302112:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204056.302127:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559937664/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204056.302138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204056.302142:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204056.302144:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204056.302150:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3187 00000100:00080000:1.0:1788204056.302154:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204056.302158:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204056.302168:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204056.302172:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559937536:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204056.302185:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559937792/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204056.302285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204056.302340:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204056/real 1788204056] req@ffff93c1c5037100 x1875064559937536/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204056.302363:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559937536/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204056.302373:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204056.302376:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204056.302380:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559937536:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204056.302386:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559937664:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204056.302403:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204056.302411:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204056/real 1788204056] req@ffff93c1c5034a80 x1875064559937664/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204056.302418:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559937664/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204056.302423:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204056.302425:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204056.302427:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559937664:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204056.302450:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559937792:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204056.302461:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204056.302467:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204056/real 1788204056] req@ffff93c1c5034e00 x1875064559937792/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204056.302474:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559937792/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204056.302482:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204056.302484:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204056.302486:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559937792:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204057.006019:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204058.029985:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204059.054039:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204060.077981:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204060.910018:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204060.910040:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204060.910171:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204060.910205:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204060.910210:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204060.910238:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204061.102075:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204061.421918:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204061.421928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204061.421931:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204061.421941:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3192 00000100:00080000:1.0:1788204061.421954:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204061.421960:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204061.421995:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559937920/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204061.422021:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204061.422024:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204061.422025:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204061.422029:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3192 00000100:00080000:1.0:1788204061.422032:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204061.422035:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204061.422044:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204061.422056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559938048/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204061.422065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204061.422070:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788204061.422071:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559937920:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204061.422074:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204061.422083:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3192 00000100:00080000:1.0:1788204061.422086:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204061.422089:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204061.422094:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204061.422105:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559938176/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204061.422150:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204061.422188:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204061/real 1788204061] req@ffff93c1c5035c00 x1875064559937920/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204061.422205:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559937920/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204061.422214:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204061.422218:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204061.422221:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559937920:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204061.422242:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559938048:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204061.422263:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204061.422275:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204061/real 1788204061] req@ffff93c1c5037800 x1875064559938048/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204061.422293:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559938048/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204061.422301:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204061.422304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204061.422307:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559938048:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204061.422312:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559938176:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204061.422330:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204061.422340:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204061/real 1788204061] req@ffff93c1c5036680 x1875064559938176/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204061.422351:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559938176/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204061.422361:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204061.422364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204061.422367:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559938176:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204062.125938:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204063.149997:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204064.173946:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204065.198026:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204066.030025:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204066.030043:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204066.030171:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204066.030202:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204066.030206:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204066.030225:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204066.221925:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204066.541928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204066.541946:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204066.541951:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204066.541963:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3197 00000100:00080000:1.0:1788204066.541976:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204066.541985:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204066.542037:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559938304/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204066.542079:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204066.542083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204066.542085:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204066.542090:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3197 00000100:00080000:1.0:1788204066.542095:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204066.542100:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204066.542111:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204066.542121:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559938304:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204066.542128:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559938432/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204066.542138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204066.542142:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204066.542144:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204066.542148:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3197 00000100:00080000:1.0:1788204066.542152:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204066.542156:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204066.542162:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204066.542173:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559938560/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204066.542191:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204066.542241:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204066/real 1788204066] req@ffff93c1c5036680 x1875064559938304/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204066.542259:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559938304/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204066.542271:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204066.542274:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204066.542278:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559938304:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204066.542311:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559938432:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204066.542332:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204066.542341:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204066/real 1788204066] req@ffff93c1c5037800 x1875064559938432/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204066.542348:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559938432/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204066.542360:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204066.542363:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204066.542366:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559938432:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204066.542372:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559938560:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204066.542390:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204066.542401:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204066/real 1788204066] req@ffff93c1c5035c00 x1875064559938560/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204066.542412:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559938560/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204066.542424:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204066.542430:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204066.542432:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559938560:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204067.246038:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204068.269927:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204069.294053:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204070.318028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204071.150078:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204071.150105:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204071.150232:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204071.150262:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204071.150266:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204071.150295:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204071.342021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204071.661985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204071.661996:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204071.662000:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204071.662009:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3202 00000100:00080000:0.0:1788204071.662020:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204071.662027:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204071.662067:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559938688/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204071.662091:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204071.662093:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204071.662095:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204071.662098:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3202 00000100:00080000:0.0:1788204071.662100:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204071.662104:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204071.662114:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204071.662125:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559938688:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204071.662128:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559938816/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204071.662141:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204071.662145:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204071.662148:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204071.662152:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3202 00000100:00080000:0.0:1788204071.662156:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204071.662160:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204071.662165:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204071.662174:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559938944/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204071.662202:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204071.662259:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204071/real 1788204071] req@ffff93c0e9ce7100 x1875064559938688/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204071.662284:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559938688/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204071.662293:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204071.662296:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204071.662300:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559938688:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204071.662323:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559938816:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204071.662340:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204071.662348:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204071/real 1788204071] req@ffff93c0e9ce4a80 x1875064559938816/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204071.662356:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559938816/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204071.662361:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204071.662364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204071.662366:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559938816:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204071.662370:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559938944:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204071.662381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204071.662386:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204071/real 1788204071] req@ffff93c0e9ce5f80 x1875064559938944/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204071.662392:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559938944/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204071.662399:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204071.662400:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204071.662402:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559938944:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204072.366037:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204073.389997:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204074.413974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204075.438046:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204076.270012:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204076.270028:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204076.270136:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204076.270161:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204076.270165:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204076.270192:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204076.461967:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204076.782006:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204076.782016:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204076.782019:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204076.782030:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3207 00000100:00080000:2.0:1788204076.782042:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204076.782049:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204076.782088:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559939072/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204076.782115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204076.782122:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204076.782124:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204076.782127:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3207 00000100:00080000:2.0:1788204076.782131:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204076.782135:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204076.782144:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204076.782157:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559939200/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204076.782161:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559939072:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204076.782176:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204076.782179:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204076.782181:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204076.782186:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3207 00000100:00080000:2.0:1788204076.782190:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204076.782194:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204076.782200:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204076.782212:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559939328/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204076.782222:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204076.782257:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204076/real 1788204076] req@ffff93c0e62bbb80 x1875064559939072/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204076.782270:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559939072/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204076.782277:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204076.782280:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204076.782282:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559939072:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204076.782300:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559939200:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204076.782316:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204076.782326:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204076/real 1788204076] req@ffff93c0e62bb100 x1875064559939200/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204076.782338:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559939200/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204076.782346:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204076.782348:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204076.782352:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559939200:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204076.782358:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559939328:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204076.782378:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204076.782389:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204076/real 1788204076] req@ffff93c0e62b8700 x1875064559939328/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204076.782400:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559939328/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204076.782416:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204076.782420:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204076.782423:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559939328:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204077.486017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204078.510006:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204079.533979:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204080.557922:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204081.390048:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204081.390061:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204081.390175:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204081.390199:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204081.390202:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204081.390229:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204081.582056:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204081.902906:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204081.902915:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204081.902920:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204081.902930:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3212 00000100:00080000:2.0:1788204081.902940:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204081.902948:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204081.902983:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559939456/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204081.903019:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204081.903023:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204081.903029:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204081.903033:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3212 00000100:00080000:2.0:1788204081.903036:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204081.903040:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204081.903048:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204081.903062:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559939584/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204081.903072:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559939456:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204081.903074:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204081.903082:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204081.903084:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204081.903088:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3212 00000100:00080000:2.0:1788204081.903091:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204081.903095:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204081.903100:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204081.903114:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559939712/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204081.903137:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204081.903184:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204081/real 1788204081] req@ffff93c0e62b8380 x1875064559939456/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204081.903204:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559939456/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204081.903213:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204081.903217:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204081.903221:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559939456:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204081.903243:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559939584:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204081.903264:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204081.903277:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204081/real 1788204081] req@ffff93c0e62b8000 x1875064559939584/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204081.903295:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559939584/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204081.903304:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204081.903307:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204081.903310:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559939584:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204081.903316:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559939712:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204081.903336:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204081.903347:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204081/real 1788204081] req@ffff93c0e62ba680 x1875064559939712/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204081.903358:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559939712/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204081.903368:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204081.903371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204081.903374:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559939712:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204082.605979:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204083.630009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204084.654003:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204085.677949:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204086.510032:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204086.510051:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204086.510149:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204086.510169:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204086.510172:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204086.510188:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204086.701996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204087.021908:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204087.021921:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204087.021927:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204087.021939:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3217 00000100:00080000:2.0:1788204087.021952:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204087.021960:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204087.022003:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559939840/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204087.022031:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204087.022035:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204087.022036:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204087.022040:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3217 00000100:00080000:2.0:1788204087.022044:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204087.022048:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204087.022060:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204087.022072:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559939968/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204087.022077:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559939840:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204087.022089:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204087.022094:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204087.022096:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204087.022101:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3217 00000100:00080000:2.0:1788204087.022105:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204087.022109:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204087.022114:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204087.022127:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559940096/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204087.022152:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204087.022209:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204087/real 1788204087] req@ffff93c0e62bb480 x1875064559939840/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204087.022228:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559939840/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204087.022244:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204087.022248:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204087.022252:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559939840:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204087.022277:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559939968:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204087.022305:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204087.022319:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204087/real 1788204087] req@ffff93c0e62baa00 x1875064559939968/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204087.022328:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559939968/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204087.022333:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204087.022335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204087.022337:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559939968:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204087.022341:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559940096:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204087.022354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204087.022360:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204087/real 1788204087] req@ffff93c0e62ba680 x1875064559940096/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204087.022367:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559940096/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204087.022373:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204087.022375:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204087.022377:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559940096:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204087.726041:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204088.750008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204089.774029:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204090.798009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204091.629940:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204091.629956:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204091.630052:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204091.630070:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204091.630072:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204091.630087:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204091.821970:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204092.141963:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204092.141974:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204092.141978:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204092.141988:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3223 00000100:00080000:2.0:1788204092.141999:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204092.142007:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204092.142055:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559940224/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204092.142078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204092.142080:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204092.142081:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204092.142086:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3223 00000100:00080000:2.0:1788204092.142088:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204092.142090:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204092.142100:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204092.142109:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559940352/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204092.142117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204092.142123:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559940224:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204092.142123:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204092.142126:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204092.142133:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3223 00000100:00080000:2.0:1788204092.142137:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204092.142141:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204092.142147:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204092.142160:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559940480/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204092.142198:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204092.142253:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204092/real 1788204092] req@ffff93c0e62b8000 x1875064559940224/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204092.142273:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559940224/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204092.142281:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204092.142284:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204092.142288:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559940224:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204092.142309:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559940352:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204092.142326:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204092.142334:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204092/real 1788204092] req@ffff93c0e62b8380 x1875064559940352/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204092.142345:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559940352/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204092.142353:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204092.142364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204092.142368:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559940352:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204092.142374:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559940480:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204092.142393:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204092.142400:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204092/real 1788204092] req@ffff93c0e62b8700 x1875064559940480/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204092.142408:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559940480/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204092.142415:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204092.142417:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204092.142419:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559940480:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204092.845929:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204093.870043:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204094.893923:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204095.918027:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204096.750025:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204096.750042:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204096.750149:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204096.750170:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204096.750173:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204096.750190:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204096.942015:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204097.261974:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204097.261989:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204097.261993:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204097.262005:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3228 00000100:00080000:2.0:1788204097.262017:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204097.262024:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204097.262065:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559940608/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204097.262102:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204097.262106:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204097.262107:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204097.262112:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3228 00000100:00080000:2.0:1788204097.262116:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204097.262119:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204097.262129:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204097.262144:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559940736/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204097.262154:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204097.262158:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204097.262160:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204097.262166:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3228 00000100:00100000:1.0:1788204097.262167:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559940608:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204097.262170:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204097.262177:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204097.262183:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204097.262196:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559940864/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204097.262243:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204097.262290:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204097/real 1788204097] req@ffff93c0e62bb100 x1875064559940608/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204097.262309:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559940608/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204097.262319:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204097.262323:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204097.262327:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559940608:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204097.262334:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559940736:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204097.262354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204097.262365:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204097/real 1788204097] req@ffff93c0e62bbb80 x1875064559940736/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204097.262381:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559940736/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204097.262395:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204097.262398:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204097.262402:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559940736:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204097.262427:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559940864:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204097.262446:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204097.262454:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204097/real 1788204097] req@ffff93c0e62bad80 x1875064559940864/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204097.262461:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559940864/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204097.262468:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204097.262470:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204097.262472:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559940864:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204097.966008:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204098.989950:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204100.014020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204101.037930:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204101.870031:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204101.870047:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204101.870159:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204101.870181:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204101.870185:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204101.870204:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204102.062039:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204102.381961:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204102.381975:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204102.381980:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204102.381992:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3233 00000100:00080000:2.0:1788204102.382005:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204102.382013:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204102.382061:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559940992/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204102.382098:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204102.382103:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204102.382105:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204102.382110:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3233 00000100:00080000:2.0:1788204102.382114:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204102.382118:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204102.382131:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204102.382148:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559941120/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204102.382161:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559940992:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204102.382163:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204102.382172:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204102.382175:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204102.382181:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3233 00000100:00080000:2.0:1788204102.382186:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204102.382191:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204102.382197:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204102.382213:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559941248/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204102.382291:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559941120:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204102.382323:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559941248:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204103.085960:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204104.110017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204105.134015:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788204105.454075:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788204105.454112:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788204105.454127:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204105.454167:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204105.454178:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:1.0:1788204105.454229:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204102/real 1788204105] req@ffff93c0e62b9c00 x1875064559941120/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204105.454274:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559941120/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204105.454291:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204105.454296:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204105.454302:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559941120:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204105.454311:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204102/real 1788204105] req@ffff93c0e62b8a80 x1875064559941248/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204105.454327:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559941248/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204105.454339:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204105.454343:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204105.454347:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559941248:192.168.201.156@tcp:8:kworker.0 00000100:00100000:1.0:1788204105.454355:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204102/real 1788204105] req@ffff93c0e62b9500 x1875064559940992/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204105.454503:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559940992/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204105.454513:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204105.454517:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204105.454521:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559940992:192.168.201.156@tcp:250:kworker.0 00010000:00010000:3.0:1788204106.158018:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204106.990056:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204106.990076:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204106.990195:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204106.990218:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204106.990222:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204106.990240:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204107.182019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204107.501909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204107.501920:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204107.501924:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204107.501934:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3238 00000100:00080000:2.0:1788204107.501946:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204107.501954:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204107.501992:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559941376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204107.502016:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204107.502018:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204107.502020:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204107.502024:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3238 00000100:00080000:2.0:1788204107.502026:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204107.502028:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204107.502039:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204107.502053:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559941504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204107.502065:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204107.502071:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1788204107.502072:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559941376:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204107.502073:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204107.502083:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3238 00000100:00080000:2.0:1788204107.502087:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204107.502091:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204107.502097:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204107.502120:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559941632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204107.502161:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204107.502212:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204107/real 1788204107] req@ffff93c0e62b9f80 x1875064559941376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204107.502234:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559941376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204107.502247:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204107.502251:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204107.502256:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559941376:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204107.502289:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559941504:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204107.502328:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204107.502359:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204107/real 1788204107] req@ffff93c0e62bb800 x1875064559941504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204107.502372:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559941504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204107.502381:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204107.502384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204107.502388:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559941504:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204107.502395:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559941632:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204107.502419:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204107.502433:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204107/real 1788204107] req@ffff93c0e62b9500 x1875064559941632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204107.502447:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559941632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204107.502465:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204107.502469:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204107.502473:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559941632:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204108.206026:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204109.230031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204110.253996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204111.277982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204112.110116:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204112.110133:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204112.110296:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204112.110336:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204112.110356:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204112.110394:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204112.302009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204112.621962:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204112.621973:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204112.621977:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204112.621988:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3243 00000100:00080000:2.0:1788204112.622001:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204112.622008:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204112.622056:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559941760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204112.622088:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204112.622098:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204112.622100:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204112.622104:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3243 00000100:00080000:2.0:1788204112.622107:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204112.622111:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204112.622122:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204112.622137:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559941888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204112.622152:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559941760:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204112.622152:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204112.622160:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204112.622164:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204112.622170:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3243 00000100:00080000:2.0:1788204112.622174:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204112.622178:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204112.622186:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204112.622201:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559942016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204112.622239:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204112.622311:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204112/real 1788204112] req@ffff93c0e62b8a80 x1875064559941760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204112.622340:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559941760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204112.622352:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204112.622356:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204112.622361:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559941760:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204112.622388:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559941888:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204112.622414:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204112.622426:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204112/real 1788204112] req@ffff93c0e62b9c00 x1875064559941888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204112.622438:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559941888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204112.622447:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204112.622450:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204112.622453:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559941888:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204112.622459:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559942016:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204112.622490:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204112.622502:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204112/real 1788204112] req@ffff93c0e62bad80 x1875064559942016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204112.622515:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559942016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204112.622526:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204112.622529:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204112.622532:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559942016:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204113.325968:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204114.350026:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204115.374011:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204116.398063:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204117.229982:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204117.230005:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204117.230118:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204117.230140:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204117.230143:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204117.230175:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204117.422026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204117.741992:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204117.742003:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204117.742006:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204117.742018:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3248 00000100:00080000:2.0:1788204117.742030:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204117.742039:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204117.742078:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559942144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204117.742101:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204117.742104:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204117.742105:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204117.742108:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3248 00000100:00080000:2.0:1788204117.742111:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204117.742114:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204117.742123:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204117.742133:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559942272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204117.742140:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204117.742143:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204117.742144:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204117.742148:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3248 00000100:00080000:2.0:1788204117.742150:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204117.742164:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204117.742170:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204117.742179:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559942144:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788204117.742184:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559942400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204117.742277:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204117.742334:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204117/real 1788204117] req@ffff93c0e62bbb80 x1875064559942144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204117.742367:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559942144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204117.742386:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204117.742392:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204117.742396:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559942144:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204117.742406:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559942272:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204117.742432:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204117.742446:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204117/real 1788204117] req@ffff93c0e62bb100 x1875064559942272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204117.742456:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559942272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204117.742463:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204117.742465:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204117.742468:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559942272:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204117.742501:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559942400:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204117.742526:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204117.742536:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204117/real 1788204117] req@ffff93c0e62b8700 x1875064559942400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204117.742546:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559942400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204117.742554:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204117.742556:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204117.742559:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559942400:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204118.446038:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204119.469973:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204120.493990:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204121.517963:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204122.350029:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204122.350045:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204122.350171:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204122.350198:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204122.350203:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204122.350229:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204122.541942:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204122.861956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204122.861964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204122.861968:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204122.861978:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3253 00000100:00080000:0.0:1788204122.861988:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204122.861994:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204122.862026:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559942528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204122.862048:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204122.862050:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204122.862052:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204122.862055:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3253 00000100:00080000:0.0:1788204122.862057:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204122.862059:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204122.862077:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204122.862085:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559942656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204122.862093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204122.862095:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204122.862097:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204122.862100:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3253 00000100:00080000:0.0:1788204122.862103:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204122.862106:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204122.862109:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204122.862116:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559942784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204122.862123:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559942528:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788204122.862187:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204122.862226:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204122/real 1788204122] req@ffff93c0e9ce7b80 x1875064559942528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204122.862241:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559942528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204122.862255:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204122.862258:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204122.862261:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559942528:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204122.862266:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559942656:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204122.862281:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204122.862289:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204122/real 1788204122] req@ffff93c0e9ce6d80 x1875064559942656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204122.862297:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559942656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204122.862302:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204122.862305:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204122.862307:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559942656:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204122.862325:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559942784:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204122.862336:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204122.862343:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204122/real 1788204122] req@ffff93c0e9ce5c00 x1875064559942784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204122.862350:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559942784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204122.862358:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204122.862359:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204122.862361:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559942784:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204123.565994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204124.589978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204125.614049:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204126.638087:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204127.470033:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204127.470049:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204127.470178:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204127.470202:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204127.470206:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204127.470231:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204127.662095:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204127.982092:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204127.982108:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204127.982115:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204127.982131:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3258 00000100:00080000:0.0:1788204127.982147:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204127.982159:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204127.982212:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559942912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204127.982245:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204127.982249:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204127.982252:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204127.982257:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3258 00000100:00080000:0.0:1788204127.982262:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204127.982267:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204127.982280:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204127.982294:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559943040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204127.982309:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559942912:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204127.982313:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204127.982326:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204127.982328:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204127.982337:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3258 00000100:00080000:0.0:1788204127.982357:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204127.982364:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204127.982371:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204127.982386:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559943168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204127.982388:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204127.982441:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204127/real 1788204127] req@ffff93c0e9ce6680 x1875064559942912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204127.982457:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559942912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204127.982468:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204127.982471:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204127.982475:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559942912:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204127.982495:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559943040:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204127.982519:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204127.982533:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204127/real 1788204127] req@ffff93c0e9ce4000 x1875064559943040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204127.982547:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559943040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204127.982557:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204127.982560:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204127.982563:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559943040:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204127.982572:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559943168:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204127.982596:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204127.982609:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204127/real 1788204127] req@ffff93c0e9ce6a00 x1875064559943168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204127.982621:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559943168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204127.982629:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204127.982631:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204127.982633:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559943168:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204128.686184:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204129.710014:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204130.734060:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204131.758004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204132.590097:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204132.590129:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204132.590286:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204132.590311:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204132.590315:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204132.590337:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204132.782011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204133.101934:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204133.101942:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204133.101946:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204133.101955:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3263 00000100:00080000:0.0:1788204133.101966:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204133.101972:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204133.102008:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559943296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204133.102038:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204133.102041:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204133.102043:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204133.102046:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3263 00000100:00080000:0.0:1788204133.102049:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204133.102052:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204133.102062:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204133.102072:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559943424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204133.102081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204133.102085:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204133.102086:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204133.102090:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3263 00000100:00080000:0.0:1788204133.102093:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204133.102095:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204133.102100:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204133.102109:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559943552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204133.102132:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559943296:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788204133.102230:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204133.102292:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204133/real 1788204133] req@ffff93c0e9ce4380 x1875064559943296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204133.102315:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559943296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204133.102326:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204133.102330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204133.102334:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559943296:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204133.102342:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559943424:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204133.102377:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204133.102388:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204133/real 1788204133] req@ffff93c0e9ce4700 x1875064559943424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204133.102398:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559943424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204133.102410:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204133.102413:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204133.102427:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559943424:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204133.102465:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559943552:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204133.102482:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204133.102490:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204133/real 1788204133] req@ffff93c0e9ce7480 x1875064559943552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204133.102500:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559943552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204133.102509:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204133.102511:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204133.102514:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559943552:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204133.806054:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204134.830090:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204135.853990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204136.878038:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204137.710019:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204137.710037:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204137.710147:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204137.710177:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204137.710183:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204137.710212:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204137.902011:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204138.221992:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204138.222006:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204138.222012:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204138.222025:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3269 00000100:00080000:2.0:1788204138.222039:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204138.222048:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204138.222097:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559943680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204138.222137:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204138.222141:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204138.222143:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204138.222148:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3269 00000100:00080000:2.0:1788204138.222152:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204138.222156:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204138.222168:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204138.222184:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559943808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204138.222197:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204138.222201:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559943680:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204138.222203:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204138.222207:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204138.222214:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3269 00000100:00080000:2.0:1788204138.222217:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204138.222221:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204138.222226:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204138.222238:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559943936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204138.222276:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204138.222341:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204138/real 1788204138] req@ffff93c0e62b8380 x1875064559943680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204138.222358:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559943680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204138.222367:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204138.222371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204138.222374:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559943680:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204138.222399:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559943808:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204138.222416:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204138.222426:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204138/real 1788204138] req@ffff93c0e62b8000 x1875064559943808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204138.222434:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559943808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204138.222440:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204138.222442:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204138.222445:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559943808:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204138.222448:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559943936:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204138.222459:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204138.222465:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204138/real 1788204138] req@ffff93c0e62ba680 x1875064559943936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204138.222473:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559943936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204138.222480:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204138.222482:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204138.222483:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559943936:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204138.926072:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204139.949948:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204140.974033:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204141.997994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204142.829988:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204142.830002:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204142.830105:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204142.830126:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204142.830128:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204142.830147:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204143.021992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204143.341975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204143.341984:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204143.341987:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204143.341995:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3274 00000100:00080000:2.0:1788204143.342005:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204143.342012:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204143.342054:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559944064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204143.342082:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204143.342086:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204143.342089:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204143.342094:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3274 00000100:00080000:2.0:1788204143.342098:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204143.342102:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204143.342111:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204143.342124:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559944192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204143.342129:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559944064:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204143.342145:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204143.342149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204143.342151:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204143.342157:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3274 00000100:00080000:2.0:1788204143.342160:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204143.342164:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204143.342171:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204143.342185:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559944320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204143.342200:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204143.342235:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204143/real 1788204143] req@ffff93c0e62baa00 x1875064559944064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204143.342249:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559944064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204143.342257:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204143.342260:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204143.342262:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559944064:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204143.342286:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559944192:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204143.342301:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204143.342310:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204143/real 1788204143] req@ffff93c0e62bb480 x1875064559944192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204143.342318:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559944192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204143.342324:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204143.342326:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204143.342328:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559944192:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204143.342332:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559944320:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204143.342343:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204143.342348:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204143/real 1788204143] req@ffff93c0e62ba680 x1875064559944320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204143.342356:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559944320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204143.342367:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204143.342369:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204143.342371:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559944320:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204144.046032:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204145.070016:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204146.094019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204147.118030:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204147.950046:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204147.950068:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204147.950194:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204147.950231:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204147.950235:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204147.950258:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204148.142006:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204148.461972:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204148.461984:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204148.461988:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204148.462010:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3279 00000100:00080000:2.0:1788204148.462023:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204148.462030:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204148.462075:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559944448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204148.462101:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204148.462104:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204148.462105:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204148.462109:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3279 00000100:00080000:2.0:1788204148.462111:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204148.462114:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204148.462124:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204148.462134:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559944576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204148.462142:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204148.462145:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204148.462146:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204148.462148:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3279 00000100:00080000:2.0:1788204148.462150:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204148.462153:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204148.462156:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204148.462165:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559944704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204148.462166:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559944448:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788204148.462279:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204148.462356:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204148/real 1788204148] req@ffff93c0e62b8000 x1875064559944448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204148.462397:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559944448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204148.462423:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204148.462428:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204148.462445:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559944448:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204148.462455:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559944576:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204148.462518:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204148.462536:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204148/real 1788204148] req@ffff93c0e62b8380 x1875064559944576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204148.462564:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559944576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204148.462590:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204148.462594:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204148.462598:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559944576:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204148.462690:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559944704:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204148.462723:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204148.462734:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204148/real 1788204148] req@ffff93c0e62b8700 x1875064559944704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204148.462761:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559944704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204148.462798:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204148.462802:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204148.462806:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559944704:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204149.166018:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204150.190030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204151.213984:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204152.237931:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204153.070011:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204153.070029:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204153.070135:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204153.070164:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204153.070168:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204153.070192:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204153.262035:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204153.581922:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204153.581933:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204153.581937:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204153.581947:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3284 00000100:00080000:3.0:1788204153.581957:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204153.581965:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204153.582014:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559944832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204153.582044:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204153.582048:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204153.582050:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204153.582055:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3284 00000100:00080000:3.0:1788204153.582059:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204153.582064:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204153.582076:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204153.582082:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559944832:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788204153.582092:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559944960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204153.582103:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204153.582106:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204153.582108:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204153.582111:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3284 00000100:00080000:3.0:1788204153.582115:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204153.582117:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204153.582122:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788204153.582130:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559945088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204153.582157:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204153.582208:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204153/real 1788204153] req@ffff93c1c4a4c380 x1875064559944832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204153.582225:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559944832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204153.582233:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204153.582236:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204153.582239:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559944832:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204153.582264:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559944960:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204153.582288:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204153.582299:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204153/real 1788204153] req@ffff93c1c4a4c700 x1875064559944960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204153.582307:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559944960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204153.582313:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204153.582315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204153.582317:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559944960:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204153.582321:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559945088:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204153.582333:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204153.582339:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204153/real 1788204153] req@ffff93c1c4a4e300 x1875064559945088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204153.582349:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559945088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204153.582359:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204153.582362:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204153.582365:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559945088:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204154.285981:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204155.310020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204156.334010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204157.358031:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204158.190046:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204158.190066:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204158.190192:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204158.190221:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204158.190226:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204158.190254:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204158.381994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204158.701978:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204158.701986:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204158.701989:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204158.701998:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3289 00000100:00080000:2.0:1788204158.702030:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204158.702043:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204158.702078:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559945216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204158.702101:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204158.702103:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204158.702104:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204158.702107:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3289 00000100:00080000:2.0:1788204158.702110:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204158.702112:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204158.702120:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204158.702133:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559945344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204158.702140:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204158.702143:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204158.702145:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204158.702149:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3289 00000100:00100000:1.0:1788204158.702152:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559945216:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204158.702153:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204158.702159:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204158.702163:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204158.702172:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559945472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204158.702209:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204158.702246:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204158/real 1788204158] req@ffff93c0e62bb100 x1875064559945216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204158.702261:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559945216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204158.702272:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204158.702275:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204158.702277:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559945216:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204158.702281:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559945344:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204158.702294:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204158.702301:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204158/real 1788204158] req@ffff93c0e62bbb80 x1875064559945344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204158.702308:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559945344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204158.702313:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204158.702314:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204158.702316:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559945344:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204158.702333:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559945472:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204158.702342:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204158.702347:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204158/real 1788204158] req@ffff93c0e62bad80 x1875064559945472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204158.702353:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559945472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204158.702359:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204158.702361:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204158.702363:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559945472:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204159.405994:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204160.429981:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204161.454005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204162.477972:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204163.310007:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204163.310020:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204163.310110:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204163.310127:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204163.310129:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204163.310144:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204163.502007:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204163.821962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204163.821978:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204163.821983:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204163.821993:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3294 00000100:00080000:0.0:1788204163.822003:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204163.822011:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204163.822048:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559945600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204163.822082:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204163.822085:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204163.822087:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204163.822092:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3294 00000100:00080000:0.0:1788204163.822096:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204163.822100:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204163.822111:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204163.822124:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559945728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204163.822134:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204163.822138:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204163.822139:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204163.822148:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559945600:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204163.822149:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3294 00000100:00080000:0.0:1788204163.822156:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204163.822163:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204163.822169:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204163.822187:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559945856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204163.822229:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204163.822280:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204163/real 1788204163] req@ffff93c0e9ce7480 x1875064559945600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204163.822297:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559945600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204163.822306:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204163.822309:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204163.822312:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559945600:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204163.822335:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559945728:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204163.822350:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204163.822381:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204163/real 1788204163] req@ffff93c0e9ce4700 x1875064559945728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204163.822390:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559945728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204163.822395:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204163.822398:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204163.822400:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559945728:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204163.822404:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559945856:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204163.822418:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204163.822424:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204163/real 1788204163] req@ffff93c0e9ce4380 x1875064559945856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204163.822432:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559945856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204163.822439:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204163.822441:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204163.822443:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559945856:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204164.525979:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204165.550014:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204166.573988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204167.598039:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204168.429999:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204168.430016:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204168.430119:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204168.430148:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204168.430151:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204168.430167:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204168.621964:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204168.941976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204168.941990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204168.941996:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204168.942009:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3299 00000100:00080000:1.0:1788204168.942022:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204168.942030:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204168.942075:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559945984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204168.942108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204168.942112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204168.942114:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204168.942119:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3299 00000100:00080000:1.0:1788204168.942123:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204168.942128:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204168.942139:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204168.942140:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559945984:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204168.942155:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559946112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204168.942167:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204168.942171:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204168.942173:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204168.942177:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3299 00000100:00080000:1.0:1788204168.942180:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204168.942184:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204168.942188:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204168.942198:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559946240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204168.942261:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559946112:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204168.942302:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559946240:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204169.646036:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204170.669930:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204171.694070:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788204172.014146:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788204172.014195:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788204172.014245:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204172.014283:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204172.014294:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788204172.014385:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204168/real 1788204172] req@ffff93c1c5034a80 x1875064559946112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204172.014434:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559946112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204172.014454:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204172.014459:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204172.014464:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559946112:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204172.014474:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204168/real 1788204172] req@ffff93c1c5037100 x1875064559946240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204172.014487:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559946240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204172.014498:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204172.014501:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204172.014504:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559946240:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788204172.014511:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204168/real 1788204172] req@ffff93c1c5034e00 x1875064559945984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204172.014524:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559945984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204172.014531:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204172.014535:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204172.014538:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559945984:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788204172.718000:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204173.550053:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204173.550069:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204173.550174:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204173.550227:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204173.550232:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204173.550254:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204173.742077:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204174.062007:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204174.062017:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204174.062021:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204174.062030:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3304 00000100:00080000:3.0:1788204174.062040:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204174.062048:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204174.062080:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559946368/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204174.062115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204174.062119:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204174.062121:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204174.062125:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3304 00000100:00080000:3.0:1788204174.062128:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204174.062132:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204174.062140:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204174.062152:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559946496/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204174.062161:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204174.062164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204174.062166:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204174.062170:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3304 00000100:00080000:3.0:1788204174.062173:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204174.062192:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204174.062197:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788204174.062208:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559946624/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204174.062274:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559946368:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788204174.062360:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204174.062425:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204174/real 1788204174] req@ffff93c1c4a4d180 x1875064559946368/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204174.062446:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559946368/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204174.062457:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204174.062467:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204174.062474:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559946368:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204174.062482:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559946496:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204174.062509:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204174.062521:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204174/real 1788204174] req@ffff93c1c4a4f800 x1875064559946496/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204174.062531:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559946496/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204174.062539:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204174.062541:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204174.062544:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559946496:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204174.062550:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559946624:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204174.062567:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204174.062578:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204174/real 1788204174] req@ffff93c1c4a4d880 x1875064559946624/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204174.062588:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559946624/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204174.062604:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204174.062607:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204174.062610:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559946624:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204174.765973:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204175.790056:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204176.814021:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204177.838017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204178.670028:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204178.670048:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204178.670171:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204178.670195:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204178.670199:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204178.670226:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204178.861930:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204179.181980:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204179.181994:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204179.181999:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204179.182012:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3310 00000100:00080000:2.0:1788204179.182025:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204179.182034:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204179.182076:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559946752/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204179.182114:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204179.182117:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204179.182118:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204179.182122:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3310 00000100:00080000:2.0:1788204179.182124:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204179.182127:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204179.182137:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204179.182148:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559946880/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204179.182160:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204179.182163:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204179.182166:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204179.182174:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3310 00000100:00100000:0.0:1788204179.182177:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559946752:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204179.182178:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204179.182189:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204179.182197:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204179.182211:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559947008/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204179.182257:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204179.182297:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204179/real 1788204179] req@ffff93c0e62b9c00 x1875064559946752/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204179.182313:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559946752/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204179.182323:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204179.182327:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204179.182331:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559946752:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204179.182357:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559946880:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204179.182378:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204179.182389:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204179/real 1788204179] req@ffff93c0e62b8a80 x1875064559946880/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204179.182401:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559946880/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204179.182415:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204179.182418:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204179.182421:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559946880:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204179.182428:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559947008:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204179.182443:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204179.182451:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204179/real 1788204179] req@ffff93c0e62b9500 x1875064559947008/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204179.182461:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559947008/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204179.182470:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204179.182473:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204179.182475:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559947008:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204179.886022:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204180.909914:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204181.934025:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204182.957999:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204183.790001:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204183.790030:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204183.790142:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204183.790179:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204183.790185:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204183.790224:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204183.982039:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204184.301973:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204184.301982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204184.301985:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204184.301994:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3315 00000100:00080000:1.0:1788204184.302014:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204184.302022:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204184.302056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559947136/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204184.302078:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204184.302081:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204184.302083:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204184.302086:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3315 00000100:00080000:1.0:1788204184.302088:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204184.302091:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204184.302099:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204184.302109:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559947264/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204184.302116:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204184.302119:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204184.302120:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204184.302122:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3315 00000100:00080000:1.0:1788204184.302125:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204184.302127:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204184.302130:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204184.302137:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559947392/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204184.302147:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559947136:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788204184.302218:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204184.302268:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204184/real 1788204184] req@ffff93c1c5034e00 x1875064559947136/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204184.302289:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559947136/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204184.302300:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204184.302304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204184.302308:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559947136:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204184.302317:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559947264:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204184.302338:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204184.302348:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204184/real 1788204184] req@ffff93c1c5037100 x1875064559947264/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204184.302357:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559947264/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204184.302364:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204184.302368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204184.302372:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559947264:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204184.302398:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559947392:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204184.302419:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204184.302429:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204184/real 1788204184] req@ffff93c1c5034a80 x1875064559947392/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204184.302437:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559947392/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204184.302445:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204184.302447:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204184.302449:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559947392:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204185.005984:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204186.029990:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204187.053989:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204188.077960:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204188.910001:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204188.910013:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204188.910105:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204188.910128:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204188.910133:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204188.910159:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204189.102051:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204189.421975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204189.421983:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204189.421985:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204189.421994:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3320 00000100:00080000:2.0:1788204189.422003:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204189.422008:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204189.422039:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559947520/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204189.422058:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204189.422061:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204189.422062:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204189.422068:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3320 00000100:00080000:2.0:1788204189.422070:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204189.422072:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204189.422080:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204189.422088:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559947648/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204189.422094:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204189.422099:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204189.422100:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204189.422106:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3320 00000100:00080000:2.0:1788204189.422109:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204189.422113:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204189.422120:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204189.422141:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559947776/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204189.422142:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559947520:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788204189.422229:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204189.422272:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204189/real 1788204189] req@ffff93c0e62bb800 x1875064559947520/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204189.422287:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559947520/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204189.422296:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204189.422299:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204189.422302:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559947520:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204189.422307:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559947648:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204189.422320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204189.422328:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204189/real 1788204189] req@ffff93c0e62b9f80 x1875064559947648/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204189.422335:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559947648/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204189.422340:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204189.422342:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204189.422343:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559947648:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204189.422366:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559947776:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204189.422377:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204189.422383:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204189/real 1788204189] req@ffff93c0e62b9500 x1875064559947776/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204189.422390:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559947776/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204189.422397:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204189.422399:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204189.422400:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559947776:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204190.126031:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204191.149996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204192.174000:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204193.198036:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204194.030060:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204194.030078:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204194.030214:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204194.030242:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204194.030246:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204194.030282:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204194.221974:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204194.541966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204194.541979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204194.541984:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204194.541996:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3325 00000100:00080000:2.0:1788204194.542018:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204194.542026:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204194.542071:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559947904/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204194.542101:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204194.542104:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204194.542106:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204194.542110:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3325 00000100:00080000:2.0:1788204194.542113:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204194.542117:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204194.542128:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204194.542139:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559948032/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204194.542149:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204194.542153:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204194.542155:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204194.542160:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3325 00000100:00080000:2.0:1788204194.542164:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204194.542168:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204194.542173:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204194.542183:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559948160/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204194.542194:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559947904:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788204194.542280:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204194.542331:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204194/real 1788204194] req@ffff93c0e62b8a80 x1875064559947904/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204194.542354:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559947904/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204194.542365:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204194.542369:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204194.542372:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559947904:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204194.542380:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559948032:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204194.542401:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204194.542413:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204194/real 1788204194] req@ffff93c0e62b9c00 x1875064559948032/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204194.542438:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559948032/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204194.542446:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204194.542449:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204194.542452:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559948032:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204194.542476:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559948160:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204194.542496:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204194.542507:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204194/real 1788204194] req@ffff93c0e62bad80 x1875064559948160/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204194.542518:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559948160/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204194.542527:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204194.542531:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204194.542534:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559948160:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204195.246009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204196.270050:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204197.294044:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204198.318024:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204199.150055:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204199.150074:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204199.150180:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204199.150200:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204199.150204:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204199.150220:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204199.341998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204199.662002:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204199.662014:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204199.662023:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204199.662036:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3330 00000100:00080000:0.0:1788204199.662048:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204199.662056:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204199.662106:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559948288/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204199.662136:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204199.662140:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204199.662142:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204199.662146:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3330 00000100:00080000:0.0:1788204199.662149:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204199.662153:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204199.662163:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204199.662175:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559948416/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204199.662185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204199.662190:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1788204199.662190:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559948288:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204199.662193:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204199.662201:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3330 00000100:00080000:0.0:1788204199.662205:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204199.662208:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204199.662213:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204199.662224:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559948544/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204199.662276:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204199.662334:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204199/real 1788204199] req@ffff93c0e9ce6a00 x1875064559948288/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204199.662356:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559948288/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204199.662375:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204199.662379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204199.662384:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559948288:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204199.662412:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559948416:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204199.662436:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204199.662457:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204199/real 1788204199] req@ffff93c0e9ce4000 x1875064559948416/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204199.662470:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559948416/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204199.662478:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204199.662482:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204199.662485:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559948416:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204199.662491:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559948544:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204199.662519:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204199.662534:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204199/real 1788204199] req@ffff93c0e9ce6680 x1875064559948544/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204199.662547:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559948544/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204199.662558:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204199.662562:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204199.662565:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559948544:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204200.365994:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204201.390041:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204202.414032:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204203.437999:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204204.269943:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204204.270019:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204204.270137:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204204.270163:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204204.270167:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204204.270195:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204204.462722:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204204.781895:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204204.781904:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204204.781907:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204204.781916:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3335 00000100:00080000:0.0:1788204204.781926:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204204.781933:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204204.781968:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559948672/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204204.781990:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204204.781992:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204204.781993:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204204.781996:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3335 00000100:00080000:0.0:1788204204.781999:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204204.782002:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204204.782010:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204204.782018:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559948800/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204204.782024:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204204.782026:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204204.782028:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204204.782030:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3335 00000100:00080000:0.0:1788204204.782032:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204204.782034:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204204.782037:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204204.782043:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559948928/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204204.782050:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559948672:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788204204.782101:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204204.782132:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204204/real 1788204204] req@ffff93c0e9ce5c00 x1875064559948672/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204204.782150:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559948672/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204204.782160:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204204.782163:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204204.782167:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559948672:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204204.782174:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559948800:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204204.782194:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204204.782206:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204204/real 1788204204] req@ffff93c0e9ce6d80 x1875064559948800/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204204.782217:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559948800/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204204.782224:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204204.782227:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204204.782230:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559948800:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204204.782254:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559948928:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204204.782271:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204204.782279:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204204/real 1788204204] req@ffff93c0e9ce7b80 x1875064559948928/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204204.782290:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559948928/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204204.782300:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204204.782303:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204204.782306:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559948928:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204205.485940:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204206.510005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204207.533941:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204208.558045:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204209.389997:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204209.390009:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204209.390098:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204209.390114:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204209.390116:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204209.390139:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204209.581965:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204209.901981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204209.901991:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204209.901996:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204209.902006:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3340 00000100:00080000:2.0:1788204209.902017:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204209.902024:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204209.902061:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559949056/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204209.902085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204209.902089:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204209.902090:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204209.902095:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3340 00000100:00080000:2.0:1788204209.902099:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204209.902103:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204209.902111:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559949056:192.168.201.156@tcp:250:kworker.0 00000008:00100000:2.0:1788204209.902113:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204209.902127:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559949184/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204209.902143:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204209.902147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204209.902148:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204209.902153:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3340 00000100:00080000:2.0:1788204209.902156:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204209.902160:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204209.902164:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204209.902176:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559949312/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204209.902183:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204209.902228:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204209/real 1788204209] req@ffff93c0e62bbb80 x1875064559949056/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204209.902243:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559949056/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204209.902250:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204209.902253:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204209.902255:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559949056:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204209.902272:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559949184:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204209.902286:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204209.902297:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204209/real 1788204209] req@ffff93c0e62bb100 x1875064559949184/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204209.902308:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559949184/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204209.902321:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204209.902324:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204209.902327:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559949184:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204209.902333:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559949312:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204209.902349:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204209.902357:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204209/real 1788204209] req@ffff93c0e62b8700 x1875064559949312/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204209.902364:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559949312/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204209.902370:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204209.902372:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204209.902374:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559949312:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204210.606007:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204211.629962:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204212.654037:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204213.678068:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204214.510096:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204214.510115:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204214.510259:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204214.510289:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204214.510294:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204214.510322:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204214.702055:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204215.022879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204215.022891:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204215.022895:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204215.022904:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3345 00000100:00080000:2.0:1788204215.022915:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204215.022923:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204215.022970:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559949440/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204215.023001:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204215.023004:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204215.023005:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204215.023009:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3345 00000100:00080000:2.0:1788204215.023012:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204215.023015:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204215.023024:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204215.023038:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559949568/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204215.023049:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204215.023053:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559949440:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204215.023054:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204215.023067:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204215.023073:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3345 00000100:00080000:2.0:1788204215.023078:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204215.023081:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204215.023086:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204215.023096:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559949696/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204215.023126:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204215.023177:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204215/real 1788204215] req@ffff93c0e62b8380 x1875064559949440/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204215.023196:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559949440/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204215.023207:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204215.023211:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204215.023215:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559949440:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204215.023242:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559949568:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204215.023264:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204215.023277:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204215/real 1788204215] req@ffff93c0e62b8000 x1875064559949568/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204215.023285:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559949568/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204215.023291:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204215.023293:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204215.023295:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559949568:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204215.023299:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559949696:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204215.023312:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204215.023318:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204215/real 1788204215] req@ffff93c0e62ba680 x1875064559949696/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204215.023325:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559949696/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204215.023332:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204215.023333:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204215.023335:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559949696:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204215.726010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204216.750375:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204217.773958:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204218.797951:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204219.629974:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204219.629987:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204219.630093:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204219.630126:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204219.630134:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204219.630175:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204219.821973:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204220.141937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204220.141944:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204220.141947:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204220.141964:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3351 00000100:00080000:2.0:1788204220.141975:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204220.141981:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204220.142019:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559949824/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204220.142047:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204220.142050:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204220.142052:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204220.142055:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3351 00000100:00080000:2.0:1788204220.142059:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204220.142062:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204220.142071:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204220.142084:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559949952/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204220.142095:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204220.142098:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204220.142101:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204220.142104:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559949824:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204220.142108:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3351 00000100:00080000:2.0:1788204220.142114:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204220.142118:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204220.142124:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204220.142138:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559950080/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204220.142168:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204220.142211:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204220/real 1788204220] req@ffff93c0e62bb480 x1875064559949824/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204220.142230:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559949824/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204220.142240:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204220.142244:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204220.142248:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559949824:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204220.142267:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559949952:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204220.142285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204220.142296:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204220/real 1788204220] req@ffff93c0e62baa00 x1875064559949952/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204220.142307:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559949952/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204220.142315:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204220.142317:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204220.142321:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559949952:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204220.142325:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559950080:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204220.142349:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204220.142361:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204220/real 1788204220] req@ffff93c0e62ba680 x1875064559950080/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204220.142373:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559950080/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204220.142397:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204220.142405:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204220.142407:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559950080:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204220.845971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204221.870110:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204222.893975:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204223.917985:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204224.749992:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204224.750008:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204224.750125:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204224.750145:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204224.750148:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204224.750168:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204224.941952:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204225.261889:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204225.261899:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204225.261903:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204225.261914:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3356 00000100:00080000:2.0:1788204225.261926:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204225.261934:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204225.261968:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559950208/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204225.261991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204225.261993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204225.261995:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204225.261998:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3356 00000100:00080000:2.0:1788204225.262000:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204225.262004:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204225.262013:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204225.262026:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559950336/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204225.262036:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204225.262039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204225.262041:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204225.262048:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3356 00000100:00100000:1.0:1788204225.262050:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559950208:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204225.262052:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204225.262060:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204225.262065:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204225.262079:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559950464/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204225.262112:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204225.262141:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204225/real 1788204225] req@ffff93c0e62b8000 x1875064559950208/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204225.262154:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559950208/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204225.262161:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204225.262163:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204225.262166:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559950208:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204225.262170:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559950336:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204225.262183:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204225.262189:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204225/real 1788204225] req@ffff93c0e62b8380 x1875064559950336/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204225.262196:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559950336/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204225.262201:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204225.262202:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204225.262204:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559950336:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204225.262221:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559950464:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204225.262231:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204225.262236:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204225/real 1788204225] req@ffff93c0e62b8700 x1875064559950464/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204225.262242:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559950464/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204225.262248:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204225.262250:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204225.262251:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559950464:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204225.966001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204226.990022:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204228.014062:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204229.037992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204229.870016:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204229.870033:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204229.870126:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204229.870145:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204229.870147:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204229.870163:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204230.062031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204230.381970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204230.381980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204230.381984:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204230.381994:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3361 00000100:00080000:2.0:1788204230.382006:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204230.382012:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204230.382061:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559950592/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204230.382093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204230.382096:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204230.382097:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204230.382101:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3361 00000100:00080000:2.0:1788204230.382103:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204230.382106:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204230.382116:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204230.382128:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559950720/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204230.382134:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204230.382137:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204230.382138:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204230.382142:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3361 00000100:00080000:2.0:1788204230.382145:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204230.382149:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204230.382153:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204230.382165:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559950592:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788204230.382166:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559950848/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204230.382275:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204230.382329:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204230/real 1788204230] req@ffff93c0e62bb100 x1875064559950592/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204230.382347:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559950592/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204230.382358:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204230.382363:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204230.382367:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559950592:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204230.382375:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559950720:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204230.382396:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204230.382409:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204230/real 1788204230] req@ffff93c0e62bbb80 x1875064559950720/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204230.382422:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559950720/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204230.382429:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204230.382432:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204230.382435:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559950720:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204230.382462:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559950848:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204230.382484:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204230.382494:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204230/real 1788204230] req@ffff93c0e62bad80 x1875064559950848/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204230.382505:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559950848/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204230.382514:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204230.382517:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204230.382520:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559950848:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204231.086030:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204232.110029:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204233.134020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204234.158034:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204234.989991:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204234.990007:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204234.990099:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204234.990138:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204234.990142:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204234.990167:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204235.181930:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204235.502009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204235.502023:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204235.502029:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204235.502042:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3366 00000100:00080000:0.0:1788204235.502055:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204235.502064:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204235.502127:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559950976/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204235.502157:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204235.502161:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204235.502163:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204235.502168:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3366 00000100:00080000:0.0:1788204235.502172:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204235.502177:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204235.502190:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204235.502203:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559950976:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204235.502206:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559951104/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204235.502227:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204235.502231:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204235.502232:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204235.502238:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3366 00000100:00080000:0.0:1788204235.502241:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204235.502245:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204235.502251:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204235.502264:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559951232/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204235.502311:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559951104:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204235.502336:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559951232:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204236.206048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204237.229975:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204238.254034:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788204238.574249:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788204238.574295:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788204238.574314:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788204238.574367:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788204238.574377:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788204238.574814:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204235/real 1788204238] req@ffff93c0e9ce4a80 x1875064559951104/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204238.574871:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559951104/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204238.574887:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204238.574892:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204238.574897:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559951104:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204238.574906:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204235/real 1788204238] req@ffff93c0e9ce7100 x1875064559951232/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204238.574922:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559951232/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204238.574939:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204238.574943:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204238.574948:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559951232:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788204238.574955:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204235/real 1788204238] req@ffff93c0e9ce5f80 x1875064559950976/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204238.574968:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559950976/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204238.574975:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204238.574978:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204238.574981:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559950976:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788204239.277976:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204240.110015:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204240.110043:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204240.110156:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204240.110184:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204240.110195:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204240.110220:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204240.302007:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204240.622055:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204240.622070:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204240.622075:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204240.622087:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3371 00000100:00080000:2.0:1788204240.622099:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204240.622108:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204240.622155:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559951360/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204240.622195:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204240.622199:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204240.622201:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204240.622205:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3371 00000100:00080000:2.0:1788204240.622209:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204240.622213:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204240.622226:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204240.622240:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559951488/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204240.622250:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559951360:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204240.622252:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204240.622261:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204240.622263:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204240.622268:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3371 00000100:00080000:2.0:1788204240.622271:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204240.622276:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204240.622282:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204240.622293:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559951616/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204240.622371:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204240.622424:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204240/real 1788204240] req@ffff93c0e62b9c00 x1875064559951360/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204240.622446:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559951360/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204240.622458:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204240.622463:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204240.622468:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559951360:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204240.622498:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559951488:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204240.622523:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204240.622536:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204240/real 1788204240] req@ffff93c0e62b8a80 x1875064559951488/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204240.622548:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559951488/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204240.622556:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204240.622559:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204240.622562:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559951488:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204240.622568:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559951616:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204240.622589:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204240.622601:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204240/real 1788204240] req@ffff93c0e62b9500 x1875064559951616/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204240.622611:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559951616/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204240.622622:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204240.622626:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204240.622629:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559951616:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204241.325966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204242.350029:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204243.373995:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204244.397972:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204245.230071:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204245.230088:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204245.230191:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204245.230214:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204245.230217:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204245.230233:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204245.421996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204245.742002:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204245.742013:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204245.742017:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204245.742027:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3376 00000100:00080000:1.0:1788204245.742038:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204245.742046:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204245.742083:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559951744/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204245.742112:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204245.742116:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204245.742118:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204245.742122:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3376 00000100:00080000:1.0:1788204245.742125:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204245.742129:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204245.742139:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204245.742151:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559951872/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204245.742163:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204245.742166:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204245.742168:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204245.742172:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559951744:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204245.742175:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3376 00000100:00080000:1.0:1788204245.742182:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204245.742187:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204245.742191:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204245.742203:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559952000/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204245.742246:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204245.742299:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204245/real 1788204245] req@ffff93c1c5035c00 x1875064559951744/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204245.742317:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559951744/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204245.742327:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204245.742331:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204245.742335:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559951744:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204245.742358:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559951872:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204245.742381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204245.742393:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204245/real 1788204245] req@ffff93c1c5037800 x1875064559951872/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204245.742404:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559951872/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204245.742412:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204245.742414:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204245.742418:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559951872:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204245.742424:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559952000:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204245.742442:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204245.742452:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204245/real 1788204245] req@ffff93c1c5036680 x1875064559952000/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204245.742463:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559952000/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204245.742477:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204245.742480:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204245.742484:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559952000:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204246.446013:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204247.470013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204248.494068:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204249.518017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204250.349975:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204250.349991:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204250.350103:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204250.350132:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204250.350137:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204250.350166:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204250.542045:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204250.862006:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204250.862019:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204250.862025:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204250.862037:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3381 00000100:00080000:1.0:1788204250.862050:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204250.862058:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204250.862110:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559952128/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204250.862148:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204250.862152:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204250.862154:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204250.862158:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3381 00000100:00080000:1.0:1788204250.862161:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204250.862165:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204250.862177:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204250.862191:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559952256/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204250.862202:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204250.862203:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559952128:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204250.862217:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204250.862219:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204250.862224:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3381 00000100:00080000:1.0:1788204250.862228:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204250.862232:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204250.862238:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204250.862251:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559952384/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204250.862287:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204250.862344:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204250/real 1788204250] req@ffff93c1c5036680 x1875064559952128/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204250.862362:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559952128/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204250.862372:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204250.862377:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204250.862381:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559952128:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204250.862406:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559952256:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204250.862426:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204250.862437:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204250/real 1788204250] req@ffff93c1c5037800 x1875064559952256/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204250.862447:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559952256/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204250.862454:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204250.862457:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204250.862460:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559952256:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204250.862466:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559952384:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204250.862480:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204250.862490:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204250/real 1788204250] req@ffff93c1c5035c00 x1875064559952384/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204250.862499:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559952384/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204250.862509:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204250.862512:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204250.862515:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559952384:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204251.566007:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204252.589961:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204253.614052:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204254.638053:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204255.470127:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204255.470147:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204255.470268:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204255.470312:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204255.470317:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204255.470353:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204255.662968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204255.982026:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204255.982041:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204255.982045:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204255.982058:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3386 00000100:00080000:0.0:1788204255.982072:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204255.982080:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204255.982130:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559952512/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204255.982163:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204255.982168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204255.982170:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204255.982174:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3386 00000100:00080000:0.0:1788204255.982178:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204255.982184:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204255.982196:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204255.982209:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559952512:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204255.982212:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559952640/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204255.982226:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204255.982230:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204255.982232:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204255.982237:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3386 00000100:00080000:0.0:1788204255.982241:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204255.982245:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204255.982250:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204255.982262:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559952768/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204255.982286:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204255.982347:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204255/real 1788204255] req@ffff93c0e9ce5f80 x1875064559952512/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204255.982377:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559952512/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204255.982388:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204255.982392:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204255.982396:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559952512:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204255.982425:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559952640:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204255.982447:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204255.982457:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204255/real 1788204255] req@ffff93c0e9ce7100 x1875064559952640/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204255.982465:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559952640/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204255.982471:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204255.982473:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204255.982475:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559952640:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204255.982479:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559952768:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204255.982495:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204255.982505:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204255/real 1788204255] req@ffff93c0e9ce4a80 x1875064559952768/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204255.982517:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559952768/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204255.982534:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204255.982536:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204255.982538:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559952768:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204256.686091:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204257.710004:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204258.734048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204259.757990:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204260.590040:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204260.590061:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204260.590163:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204260.590183:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204260.590186:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204260.590201:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204260.782034:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204261.101933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204261.101947:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204261.101952:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204261.101971:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3391 00000100:00080000:2.0:1788204261.101984:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204261.101991:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204261.102037:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559952896/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204261.102070:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204261.102073:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204261.102075:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204261.102078:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3391 00000100:00080000:2.0:1788204261.102081:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204261.102084:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204261.102094:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204261.102104:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559953024/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204261.102112:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204261.102114:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204261.102116:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204261.102121:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3391 00000100:00080000:2.0:1788204261.102123:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788204261.102124:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559952896:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204261.102125:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204261.102131:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204261.102140:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559953152/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204261.102205:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204261.102254:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204261/real 1788204261] req@ffff93c0e62b9f80 x1875064559952896/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204261.102273:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559952896/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204261.102284:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204261.102289:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204261.102292:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559952896:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204261.102319:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559953024:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204261.102336:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204261.102345:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204261/real 1788204261] req@ffff93c0e62bb800 x1875064559953024/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204261.102352:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559953024/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204261.102358:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204261.102360:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204261.102361:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559953024:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204261.102366:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559953152:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204261.102376:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204261.102381:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204261/real 1788204261] req@ffff93c0e62b9500 x1875064559953152/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204261.102388:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559953152/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204261.102395:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204261.102397:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204261.102399:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559953152:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204261.805992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204262.830058:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204263.854007:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204264.878060:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204265.710038:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204265.710054:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204265.710157:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204265.710199:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204265.710204:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204265.710233:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204265.902025:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204266.221970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204266.221985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204266.221991:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204266.222004:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3397 00000100:00080000:2.0:1788204266.222021:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204266.222030:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204266.222074:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559953280/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204266.222104:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204266.222107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204266.222109:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204266.222112:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3397 00000100:00080000:2.0:1788204266.222115:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204266.222118:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204266.222128:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204266.222136:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559953408/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204266.222144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204266.222149:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559953280:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204266.222149:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204266.222150:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204266.222158:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3397 00000100:00080000:2.0:1788204266.222161:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204266.222163:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204266.222166:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204266.222176:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559953536/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204266.222225:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204266.222273:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204266/real 1788204266] req@ffff93c0e62b8a80 x1875064559953280/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204266.222287:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559953280/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204266.222298:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204266.222302:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204266.222306:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559953280:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204266.222333:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559953408:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204266.222351:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204266.222360:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204266/real 1788204266] req@ffff93c0e62b9c00 x1875064559953408/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204266.222368:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559953408/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204266.222374:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204266.222376:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204266.222378:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559953408:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204266.222382:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559953536:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204266.222392:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204266.222398:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204266/real 1788204266] req@ffff93c0e62bad80 x1875064559953536/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204266.222405:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559953536/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204266.222411:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204266.222413:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204266.222415:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559953536:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204266.926020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204267.949998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204268.974083:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204269.997956:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204270.830017:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204270.830037:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204270.830151:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204270.830171:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204270.830174:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204270.830191:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204271.021940:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204271.341910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204271.341924:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204271.341929:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204271.341943:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3402 00000100:00080000:2.0:1788204271.341957:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204271.341965:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204271.342013:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559953664/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204271.342037:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204271.342040:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204271.342041:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204271.342044:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3402 00000100:00080000:2.0:1788204271.342047:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204271.342050:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204271.342059:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204271.342068:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559953792/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204271.342084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204271.342088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1788204271.342089:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559953664:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204271.342091:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204271.342101:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3402 00000100:00080000:2.0:1788204271.342105:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204271.342108:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204271.342122:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204271.342136:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559953920/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204271.342167:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204271.342207:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204271/real 1788204271] req@ffff93c0e62bbb80 x1875064559953664/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204271.342226:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559953664/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204271.342244:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204271.342248:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204271.342252:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559953664:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204271.342278:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559953792:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204271.342307:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204271.342317:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204271/real 1788204271] req@ffff93c0e62bb100 x1875064559953792/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204271.342325:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559953792/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204271.342331:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204271.342333:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204271.342335:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559953792:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204271.342339:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559953920:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204271.342350:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204271.342356:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204271/real 1788204271] req@ffff93c0e62b8700 x1875064559953920/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204271.342362:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559953920/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204271.342369:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204271.342371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204271.342372:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559953920:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204272.045973:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204273.070066:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204274.094067:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204275.118051:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204275.950131:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204275.950151:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204275.950250:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204275.950271:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204275.950274:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204275.950290:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204276.142036:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204276.461974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204276.461990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204276.461995:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204276.462007:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3407 00000100:00080000:0.0:1788204276.462018:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204276.462025:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204276.462073:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559954048/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204276.462103:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204276.462106:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204276.462107:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204276.462111:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3407 00000100:00080000:0.0:1788204276.462114:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204276.462119:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204276.462130:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204276.462147:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559954176/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204276.462148:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559954048:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204276.462171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204276.462175:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204276.462177:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204276.462182:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3407 00000100:00080000:0.0:1788204276.462186:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204276.462190:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204276.462195:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204276.462205:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559954304/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204276.462219:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204276.462266:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204276/real 1788204276] req@ffff93c0e9ce5180 x1875064559954048/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204276.462285:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559954048/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204276.462296:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204276.462300:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204276.462304:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559954048:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204276.462329:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559954176:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204276.462353:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204276.462365:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204276/real 1788204276] req@ffff93c0e9ce5500 x1875064559954176/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204276.462372:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559954176/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204276.462377:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204276.462380:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204276.462382:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559954176:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204276.462385:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559954304:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204276.462414:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204276.462426:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204276/real 1788204276] req@ffff93c0e9ce7800 x1875064559954304/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204276.462438:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559954304/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204276.462449:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204276.462451:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204276.462453:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559954304:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204277.166032:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204278.189988:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204279.215291:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204280.238001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204281.070034:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204281.070057:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204281.070163:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204281.070185:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204281.070188:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204281.070204:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204281.262009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204281.581949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204281.581957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204281.581960:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204281.581970:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3412 00000100:00080000:0.0:1788204281.581982:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204281.581990:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204281.582030:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559954432/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204281.582059:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204281.582063:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204281.582065:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204281.582069:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3412 00000100:00080000:0.0:1788204281.582072:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204281.582076:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204281.582086:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204281.582098:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559954560/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204281.582106:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204281.582108:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204281.582110:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204281.582110:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559954432:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204281.582115:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3412 00000100:00080000:0.0:1788204281.582119:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204281.582124:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204281.582127:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204281.582134:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559954688/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204281.582199:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204281.582268:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204281/real 1788204281] req@ffff93c0e9ce4e00 x1875064559954432/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204281.582293:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559954432/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204281.582305:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204281.582350:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204281.582355:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559954432:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204281.582386:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559954560:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204281.582418:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204281.582431:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204281/real 1788204281] req@ffff93c0e9ce5880 x1875064559954560/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204281.582442:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559954560/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204281.582454:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204281.582457:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204281.582460:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559954560:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204281.582466:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559954688:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204281.582486:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204281.582497:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204281/real 1788204281] req@ffff93c0e9ce6300 x1875064559954688/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204281.582507:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559954688/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204281.582517:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204281.582520:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204281.582523:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559954688:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204282.285955:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204283.310030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204284.333957:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204285.358056:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204286.190045:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204286.190063:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204286.190176:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204286.190199:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204286.190202:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204286.190220:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204286.382015:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204286.701947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204286.701957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204286.701961:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204286.701971:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3417 00000100:00080000:0.0:1788204286.701984:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204286.701992:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204286.702043:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559954816/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204286.702069:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204286.702072:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204286.702073:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204286.702077:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3417 00000100:00080000:0.0:1788204286.702080:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204286.702083:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204286.702093:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204286.702102:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559954944/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204286.702110:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204286.702112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204286.702115:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204286.702118:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3417 00000100:00080000:0.0:1788204286.702120:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204286.702123:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204286.702126:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204286.702133:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559955072/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204286.702144:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559954816:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788204286.702215:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204286.702268:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204286/real 1788204286] req@ffff93c0e9ce6300 x1875064559954816/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204286.702281:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559954816/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204286.702289:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204286.702293:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204286.702296:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559954816:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204286.702302:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559954944:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204286.702319:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204286.702326:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204286/real 1788204286] req@ffff93c0e9ce5880 x1875064559954944/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204286.702333:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559954944/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204286.702338:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204286.702340:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204286.702342:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559954944:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204286.702360:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559955072:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204286.702371:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204286.702378:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204286/real 1788204286] req@ffff93c0e9ce4e00 x1875064559955072/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204286.702389:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559955072/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204286.702399:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204286.702403:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204286.702406:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559955072:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204287.405984:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204288.429956:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204289.453980:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204290.477940:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204291.310051:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204291.310069:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204291.310191:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204291.310220:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204291.310231:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204291.310253:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204291.501975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204291.821936:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204291.821946:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204291.821950:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204291.821960:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3422 00000100:00080000:0.0:1788204291.821972:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204291.821979:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204291.822026:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559955200/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204291.822066:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204291.822069:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204291.822070:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204291.822074:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3422 00000100:00080000:0.0:1788204291.822077:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204291.822082:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204291.822094:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204291.822106:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559955200:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204291.822109:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559955328/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204291.822120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204291.822124:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204291.822126:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204291.822130:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3422 00000100:00080000:0.0:1788204291.822152:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204291.822156:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204291.822161:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204291.822175:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559955456/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204291.822212:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204291.822261:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204291/real 1788204291] req@ffff93c0e9ce7800 x1875064559955200/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204291.822277:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559955200/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204291.822285:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204291.822289:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204291.822293:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559955200:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204291.822314:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559955328:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204291.822330:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204291.822338:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204291/real 1788204291] req@ffff93c0e9ce5500 x1875064559955328/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204291.822349:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559955328/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204291.822357:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204291.822360:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204291.822364:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559955328:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204291.822370:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559955456:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204291.822390:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204291.822401:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204291/real 1788204291] req@ffff93c0e9ce5180 x1875064559955456/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204291.822413:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559955456/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204291.822428:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204291.822430:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204291.822432:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559955456:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204292.526010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204293.549961:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204294.574017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204295.597965:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204296.430127:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204296.430141:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204296.430255:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204296.430282:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204296.430287:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204296.430314:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204296.622011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204296.941982:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204296.941995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204296.942001:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204296.942014:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3427 00000100:00080000:2.0:1788204296.942026:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204296.942034:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204296.942090:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559955584/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204296.942128:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204296.942136:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204296.942137:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204296.942142:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3427 00000100:00080000:2.0:1788204296.942145:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204296.942149:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204296.942161:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204296.942181:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559955584:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788204296.942181:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559955712/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204296.942196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204296.942201:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204296.942203:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204296.942208:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3427 00000100:00080000:2.0:1788204296.942211:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204296.942216:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204296.942222:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204296.942235:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559955840/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204296.942276:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204296.942333:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204296/real 1788204296] req@ffff93c0e62b8380 x1875064559955584/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204296.942355:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559955584/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204296.942365:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204296.942370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204296.942372:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559955584:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204296.942403:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559955712:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204296.942422:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204296.942431:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204296/real 1788204296] req@ffff93c0e62b8000 x1875064559955712/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204296.942439:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559955712/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204296.942444:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204296.942446:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204296.942448:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559955712:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204296.942452:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559955840:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204296.942464:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204296.942472:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204296/real 1788204296] req@ffff93c0e62ba680 x1875064559955840/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204296.942483:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559955840/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204296.942493:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204296.942496:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204296.942499:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559955840:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204297.646005:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204298.669935:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204299.694043:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204300.717960:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204301.550002:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204301.550019:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204301.550136:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204301.550159:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204301.550162:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204301.550178:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204301.742038:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204302.061944:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204302.061969:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204302.061975:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204302.061988:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3432 00000100:00080000:1.0:1788204302.062001:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204302.062008:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204302.062050:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559955968/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204302.062083:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204302.062085:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204302.062087:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204302.062090:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3432 00000100:00080000:1.0:1788204302.062092:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204302.062095:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204302.062104:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204302.062113:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559956096/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204302.062120:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204302.062122:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204302.062123:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204302.062126:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3432 00000100:00080000:1.0:1788204302.062134:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204302.062135:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559955968:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204302.062141:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204302.062148:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204302.062159:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559956224/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204302.062257:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559956096:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204302.062300:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559956224:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204302.765997:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204303.790046:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204304.814077:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788204305.134219:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:1.0:1788204305.134303:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788204305.134327:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788204305.134397:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:1.0:1788204305.134412:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788204305.134465:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204302/real 1788204305] req@ffff93c1c5037100 x1875064559956096/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204305.134497:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559956096/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204305.134510:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204305.134515:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204305.134534:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559956096:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204305.134543:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204302/real 1788204305] req@ffff93c1c5034e00 x1875064559956224/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204305.134556:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559956224/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204305.134567:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204305.134571:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204305.134575:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559956224:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788204305.134582:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204302/real 1788204305] req@ffff93c1c5034a80 x1875064559955968/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204305.134595:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559955968/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204305.134603:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204305.134606:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204305.134610:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559955968:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788204305.838026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204306.670013:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204306.670034:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204306.670161:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204306.670191:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204306.670196:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204306.670228:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204306.862012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204307.181982:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204307.181998:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204307.182003:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204307.182015:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3438 00000100:00080000:2.0:1788204307.182033:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204307.182040:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204307.182090:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559956352/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204307.182120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204307.182124:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204307.182130:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204307.182135:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3438 00000100:00080000:2.0:1788204307.182139:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204307.182143:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204307.182155:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204307.182168:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559956480/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204307.182179:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559956352:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204307.182181:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204307.182189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204307.182191:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204307.182196:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3438 00000100:00080000:2.0:1788204307.182199:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204307.182203:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204307.182209:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204307.182221:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559956608/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204307.182272:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204307.182326:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204307/real 1788204307] req@ffff93c0e62baa00 x1875064559956352/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204307.182347:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559956352/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204307.182359:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204307.182364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204307.182368:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559956352:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204307.182403:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559956480:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204307.182434:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204307.182448:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204307/real 1788204307] req@ffff93c0e62bb480 x1875064559956480/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204307.182460:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559956480/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204307.182468:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204307.182472:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204307.182475:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559956480:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204307.182481:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559956608:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204307.182501:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204307.182511:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204307/real 1788204307] req@ffff93c0e62ba680 x1875064559956608/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204307.182522:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559956608/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204307.182532:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204307.182535:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204307.182538:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559956608:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204307.886032:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204308.909987:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204309.933991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204310.957908:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204311.790050:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204311.790066:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204311.790195:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204311.790221:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204311.790225:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204311.790251:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204311.981976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204312.301910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204312.301926:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204312.301930:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204312.301942:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3443 00000100:00080000:2.0:1788204312.301955:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204312.301964:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204312.302010:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559956736/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204312.302046:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204312.302050:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204312.302052:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204312.302056:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3443 00000100:00080000:2.0:1788204312.302060:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204312.302065:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204312.302076:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204312.302092:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559956736:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788204312.302093:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559956864/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204312.302108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204312.302112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204312.302114:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204312.302119:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3443 00000100:00080000:2.0:1788204312.302123:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204312.302127:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204312.302133:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204312.302146:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559956992/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204312.302168:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204312.302208:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204312/real 1788204312] req@ffff93c0e62b8000 x1875064559956736/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204312.302222:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559956736/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204312.302230:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204312.302233:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204312.302236:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559956736:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204312.302256:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559956864:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204312.302270:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204312.302277:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204312/real 1788204312] req@ffff93c0e62b8380 x1875064559956864/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204312.302285:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559956864/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204312.302290:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204312.302292:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204312.302294:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559956864:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204312.302297:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559956992:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204312.302306:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204312.302311:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204312/real 1788204312] req@ffff93c0e62b8700 x1875064559956992/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204312.302318:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559956992/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204312.302329:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204312.302331:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204312.302333:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559956992:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204313.005925:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204314.029980:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204315.053961:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204316.077962:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204316.909976:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204316.909992:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204316.910099:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204316.910123:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204316.910127:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204316.910153:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204317.101960:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204317.421929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204317.421940:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204317.421944:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204317.421955:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3448 00000100:00080000:2.0:1788204317.421966:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204317.421974:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204317.422008:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559957120/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204317.422036:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204317.422048:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204317.422050:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204317.422054:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3448 00000100:00080000:2.0:1788204317.422057:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204317.422062:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204317.422072:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204317.422084:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559957248/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204317.422093:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559957120:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204317.422099:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204317.422104:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204317.422106:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204317.422111:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3448 00000100:00080000:2.0:1788204317.422115:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204317.422118:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204317.422132:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204317.422144:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559957376/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204317.422161:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204317.422198:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204317/real 1788204317] req@ffff93c0e62bb100 x1875064559957120/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204317.422215:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559957120/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204317.422225:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204317.422229:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204317.422233:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559957120:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204317.422254:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559957248:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204317.422275:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204317.422288:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204317/real 1788204317] req@ffff93c0e62bbb80 x1875064559957248/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204317.422300:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559957248/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204317.422308:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204317.422312:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204317.422315:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559957248:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204317.422321:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559957376:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204317.422338:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204317.422348:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204317/real 1788204317] req@ffff93c0e62bad80 x1875064559957376/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204317.422358:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559957376/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204317.422368:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204317.422371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204317.422374:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559957376:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204318.125990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204319.149963:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204320.173923:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204321.197965:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204322.029952:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204322.029964:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204322.030065:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204322.030084:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204322.030087:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204322.030104:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204322.221944:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204322.541970:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204322.541981:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204322.541986:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204322.541997:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3453 00000100:00080000:0.0:1788204322.542008:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204322.542017:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204322.542056:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559957504/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204322.542083:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204322.542086:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204322.542088:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204322.542091:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3453 00000100:00080000:0.0:1788204322.542094:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204322.542098:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204322.542106:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204322.542117:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559957632/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204322.542126:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204322.542127:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559957504:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204322.542132:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204322.542136:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204322.542140:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3453 00000100:00080000:0.0:1788204322.542143:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204322.542145:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204322.542149:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204322.542158:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559957760/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204322.542194:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204322.542247:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204322/real 1788204322] req@ffff93c0e9ce4a80 x1875064559957504/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204322.542266:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559957504/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204322.542274:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204322.542277:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204322.542280:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559957504:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204322.542300:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559957632:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204322.542318:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204322.542328:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204322/real 1788204322] req@ffff93c0e9ce7100 x1875064559957632/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204322.542336:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559957632/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204322.542343:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204322.542346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204322.542349:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559957632:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204322.542356:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559957760:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204322.542376:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204322.542388:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204322/real 1788204322] req@ffff93c0e9ce5f80 x1875064559957760/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204322.542401:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559957760/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204322.542409:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204322.542411:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204322.542413:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559957760:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204323.245971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204324.269979:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204325.293974:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204326.318031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204327.150060:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204327.150081:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204327.150217:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204327.150267:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204327.150273:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204327.150382:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204327.342054:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204327.661962:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204327.661979:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204327.661985:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204327.662000:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3458 00000100:00080000:3.0:1788204327.662014:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204327.662024:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204327.662075:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d500 x1875064559957888/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204327.662110:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204327.662114:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204327.662116:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204327.662121:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3458 00000100:00080000:3.0:1788204327.662124:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204327.662129:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204327.662141:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204327.662156:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ca80 x1875064559958016/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204327.662166:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559957888:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788204327.662170:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204327.662178:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204327.662180:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204327.662185:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3458 00000100:00080000:3.0:1788204327.662189:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204327.662193:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204327.662199:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788204327.662222:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559958144/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204327.662264:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204327.662326:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204327/real 1788204327] req@ffff93c1c4a4d500 x1875064559957888/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204327.662344:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d500 x1875064559957888/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204327.662354:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204327.662358:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204327.662362:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559957888:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204327.662393:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559958016:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204327.662411:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204327.662421:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204327/real 1788204327] req@ffff93c1c4a4ca80 x1875064559958016/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204327.662429:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ca80 x1875064559958016/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204327.662440:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204327.662443:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204327.662445:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559958016:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204327.662451:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559958144:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204327.662471:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204327.662483:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204327/real 1788204327] req@ffff93c1c4a4d880 x1875064559958144/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204327.662492:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559958144/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204327.662500:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204327.662503:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204327.662505:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559958144:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204328.366051:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204329.390001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204330.413993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204331.437956:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204332.270032:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204332.270050:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204332.270147:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204332.270168:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204332.270171:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204332.270186:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204332.461950:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204332.781975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204332.781993:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204332.781999:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204332.782014:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3463 00000100:00080000:0.0:1788204332.782028:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204332.782037:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204332.782215:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559958272/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204332.782246:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204332.782250:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204332.782253:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204332.782257:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3463 00000100:00080000:0.0:1788204332.782266:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204332.782275:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204332.782285:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204332.782301:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559958272:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204332.782311:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559958400/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204332.782326:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204332.782330:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204332.782333:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204332.782338:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3463 00000100:00080000:0.0:1788204332.782343:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204332.782347:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204332.782354:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204332.782369:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559958528/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204332.782390:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204332.782468:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204332/real 1788204332] req@ffff93c0e9ce7b80 x1875064559958272/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204332.782492:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559958272/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204332.782504:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204332.782508:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204332.782512:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559958272:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204332.782536:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559958400:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204332.782555:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204332.782573:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204332/real 1788204332] req@ffff93c0e9ce6d80 x1875064559958400/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204332.782581:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559958400/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204332.782586:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204332.782589:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204332.782591:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559958400:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204332.782595:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559958528:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204332.782614:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204332.782625:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204332/real 1788204332] req@ffff93c0e9ce5c00 x1875064559958528/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204332.782636:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559958528/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204332.782648:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204332.782651:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204332.782655:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559958528:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204333.486021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204334.509996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204335.534013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204336.558001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204337.390027:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204337.390044:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204337.390301:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204337.390404:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204337.390415:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204337.390467:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204337.582082:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204337.901934:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204337.901942:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204337.901946:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204337.901954:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3468 00000100:00080000:0.0:1788204337.901963:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204337.901969:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204337.902004:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559958656/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204337.902028:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204337.902031:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204337.902032:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204337.902036:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3468 00000100:00080000:0.0:1788204337.902040:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204337.902043:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204337.902053:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204337.902068:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559958784/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204337.902092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204337.902096:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204337.902098:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204337.902104:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3468 00000100:00080000:0.0:1788204337.902109:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788204337.902115:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559958656:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204337.902117:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204337.902130:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204337.902151:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559958912/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204337.902227:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204337.902293:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204337/real 1788204337] req@ffff93c0e9ce6680 x1875064559958656/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204337.902315:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559958656/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204337.902346:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204337.902352:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204337.902357:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559958656:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204337.902366:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559958784:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204337.902391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204337.902405:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204337/real 1788204337] req@ffff93c0e9ce4000 x1875064559958784/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204337.902418:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559958784/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204337.902427:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204337.902430:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204337.902433:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559958784:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204337.902473:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559958912:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204337.902495:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204337.902506:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204337/real 1788204337] req@ffff93c0e9ce6a00 x1875064559958912/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204337.902524:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559958912/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204337.902540:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204337.902544:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204337.902547:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559958912:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204338.605975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204339.629982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204340.653997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204341.678016:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204342.509999:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204342.510018:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204342.510137:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204342.510169:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204342.510174:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204342.510209:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204342.701994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204343.021883:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204343.021894:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204343.021898:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204343.021909:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3473 00000100:00080000:0.0:1788204343.021920:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204343.021928:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204343.021966:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559959040/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204343.021991:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204343.021995:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204343.021997:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204343.022001:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3473 00000100:00080000:0.0:1788204343.022005:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204343.022009:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204343.022017:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204343.022028:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559959168/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204343.022036:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204343.022039:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1788204343.022039:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559959040:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204343.022042:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204343.022049:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3473 00000100:00080000:0.0:1788204343.022052:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204343.022054:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204343.022058:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204343.022067:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559959296/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204343.022116:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204343.022157:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204343/real 1788204343] req@ffff93c0e9ce4380 x1875064559959040/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204343.022176:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559959040/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204343.022187:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204343.022191:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204343.022195:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559959040:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204343.022220:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559959168:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204343.022260:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204343.022276:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204343/real 1788204343] req@ffff93c0e9ce4700 x1875064559959168/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204343.022290:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559959168/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204343.022300:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204343.022303:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204343.022307:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559959168:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204343.022314:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559959296:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204343.022338:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204343.022351:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204343/real 1788204343] req@ffff93c0e9ce7480 x1875064559959296/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204343.022364:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559959296/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204343.022375:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204343.022379:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204343.022382:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559959296:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204343.726140:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204344.750043:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204345.774090:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204346.798014:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204347.630101:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204347.630116:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204347.630216:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204347.630235:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204347.630238:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204347.630255:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204347.822003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204348.141950:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204348.141961:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204348.141964:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204348.141975:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3479 00000100:00080000:0.0:1788204348.141987:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204348.141995:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204348.142050:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559959424/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204348.142079:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204348.142083:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204348.142085:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204348.142089:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3479 00000100:00080000:0.0:1788204348.142093:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204348.142096:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204348.142106:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204348.142120:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559959552/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204348.142130:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204348.142132:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559959424:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204348.142135:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204348.142139:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204348.142145:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3479 00000100:00080000:0.0:1788204348.142148:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204348.142151:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204348.142170:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204348.142185:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559959680/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204348.142205:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204348.142251:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204348/real 1788204348] req@ffff93c0e9ce7480 x1875064559959424/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204348.142272:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559959424/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204348.142283:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204348.142286:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204348.142288:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559959424:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204348.142306:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559959552:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204348.142323:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204348.142331:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204348/real 1788204348] req@ffff93c0e9ce4700 x1875064559959552/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204348.142339:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559959552/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204348.142343:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204348.142345:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204348.142347:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559959552:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204348.142350:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559959680:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204348.142361:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204348.142368:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204348/real 1788204348] req@ffff93c0e9ce4380 x1875064559959680/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204348.142378:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559959680/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204348.142387:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204348.142390:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204348.142393:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559959680:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204348.846068:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204349.869973:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204350.893997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204351.918083:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204352.750036:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204352.750048:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204352.750179:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204352.750199:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204352.750202:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204352.750220:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204352.941999:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204353.262022:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204353.262029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204353.262032:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204353.262040:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3484 00000100:00080000:3.0:1788204353.262050:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204353.262057:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204353.262118:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559959808/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204353.262145:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204353.262147:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204353.262149:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204353.262152:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3484 00000100:00080000:3.0:1788204353.262155:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204353.262158:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204353.262165:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204353.262175:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559959936/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204353.262181:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204353.262184:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204353.262185:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204353.262188:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3484 00000100:00080000:3.0:1788204353.262189:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204353.262191:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204353.262195:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204353.262199:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559959808:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788204353.262207:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559960064/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204353.262262:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204353.262300:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204353/real 1788204353] req@ffff93c1c4a4f800 x1875064559959808/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204353.262316:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559959808/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204353.262332:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204353.262336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204353.262341:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559959808:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204353.262348:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559959936:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204353.262368:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204353.262380:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204353/real 1788204353] req@ffff93c1c4a4d180 x1875064559959936/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204353.262392:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559959936/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204353.262401:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204353.262404:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204353.262408:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559959936:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204353.262439:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559960064:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204353.262460:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204353.262472:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204353/real 1788204353] req@ffff93c1c4a4e300 x1875064559960064/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204353.262483:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559960064/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204353.262493:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204353.262496:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204353.262499:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559960064:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204353.966038:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204354.990006:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204356.014005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204357.037998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204357.870036:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204357.870051:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204357.870158:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204357.870188:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204357.870192:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204357.870222:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204358.062013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204358.381944:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204358.381954:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204358.381958:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204358.381967:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3489 00000100:00080000:3.0:1788204358.382006:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204358.382015:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204358.382049:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559960192/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204358.382078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204358.382086:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204358.382087:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204358.382091:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3489 00000100:00080000:3.0:1788204358.382095:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204358.382099:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204358.382108:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204358.382121:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559960320/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204358.382123:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559960192:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788204358.382138:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204358.382141:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204358.382144:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204358.382148:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3489 00000100:00080000:3.0:1788204358.382152:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204358.382156:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204358.382161:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000800:00000100:1.0:1788204358.382182:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:3.0:1788204358.382212:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559960448/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204358.382224:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204358/real 1788204358] req@ffff93c1c4a4c700 x1875064559960192/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204358.382237:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559960192/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204358.382245:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204358.382248:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204358.382251:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559960192:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204358.382268:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559960320:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204358.382282:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204358.382289:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204358/real 1788204358] req@ffff93c1c4a4c380 x1875064559960320/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204358.382296:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559960320/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204358.382301:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204358.382303:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204358.382305:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559960320:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204358.382308:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559960448:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204358.382317:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204358.382322:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204358/real 1788204358] req@ffff93c1c4a4f480 x1875064559960448/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204358.382329:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559960448/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204358.382334:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204358.382336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204358.382337:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559960448:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204359.085927:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204360.110115:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204361.134001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204362.158074:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204362.990061:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204362.990113:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204362.990211:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204362.990237:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204362.990243:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204362.990275:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204363.181991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204363.501995:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204363.502007:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204363.502011:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204363.502022:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3494 00000100:00080000:2.0:1788204363.502033:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204363.502040:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204363.502075:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559960576/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204363.502102:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204363.502105:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204363.502107:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204363.502111:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3494 00000100:00080000:2.0:1788204363.502114:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204363.502117:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204363.502126:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204363.502138:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559960704/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204363.502149:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204363.502152:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204363.502154:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204363.502157:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559960576:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204363.502159:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3494 00000100:00080000:2.0:1788204363.502165:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204363.502171:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204363.502221:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000800:00000100:1.0:1788204363.502224:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788204363.502238:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559960832/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204363.502268:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204363/real 1788204363] req@ffff93c0e62b9c00 x1875064559960576/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204363.502286:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559960576/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204363.502296:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204363.502300:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204363.502304:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559960576:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204363.502327:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559960704:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204363.502348:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204363.502359:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204363/real 1788204363] req@ffff93c0e62b8a80 x1875064559960704/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204363.502369:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559960704/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204363.502383:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204363.502386:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204363.502389:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559960704:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204363.502394:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559960832:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204363.502412:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204363.502422:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204363/real 1788204363] req@ffff93c0e62b9500 x1875064559960832/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204363.502433:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559960832/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204363.502442:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204363.502445:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204363.502448:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559960832:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204364.206037:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204365.230002:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204366.254069:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204367.277960:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204368.110067:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204368.110088:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204368.110242:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204368.110272:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204368.110277:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204368.110313:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204368.302080:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204368.621994:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204368.622000:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204368.622003:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204368.622011:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3499 00000100:00080000:0.0:1788204368.622019:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204368.622025:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204368.622093:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559960960/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204368.622117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204368.622119:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204368.622121:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204368.622123:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3499 00000100:00080000:0.0:1788204368.622126:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204368.622129:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204368.622136:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204368.622144:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559961088/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204368.622155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204368.622158:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204368.622159:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204368.622163:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3499 00000100:00080000:0.0:1788204368.622164:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204368.622167:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204368.622171:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204368.622177:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559961216/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204368.622188:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559960960:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204368.622325:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559961088:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204368.622383:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559961216:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204369.325962:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204370.350037:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204371.373969:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788204371.694183:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788204371.694241:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788204371.694261:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204371.694310:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204371.694320:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:1.0:1788204371.694384:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204368/real 1788204371] req@ffff93c0e9ce6680 x1875064559961216/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204371.694414:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559961216/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204371.694434:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204371.694440:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204371.694445:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559961216:192.168.201.156@tcp:8:kworker.0 00000100:00100000:1.0:1788204371.694453:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204368/real 1788204371] req@ffff93c0e9ce6a00 x1875064559960960/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204371.694467:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559960960/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204371.694473:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204371.694477:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204371.694480:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559960960:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204371.694488:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204368/real 1788204371] req@ffff93c0e9ce4000 x1875064559961088/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204371.694500:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559961088/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204371.694508:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204371.694512:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204371.694515:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559961088:192.168.201.156@tcp:38:kworker.0 00010000:00010000:3.0:1788204372.397980:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204373.229965:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204373.229981:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204373.230063:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204373.230080:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204373.230083:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204373.230098:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204373.421994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204373.741916:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204373.741931:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204373.741936:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204373.741949:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3504 00000100:00080000:0.0:1788204373.741962:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204373.741972:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204373.742015:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559961344/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204373.742057:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204373.742060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204373.742061:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204373.742065:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3504 00000100:00080000:0.0:1788204373.742067:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204373.742070:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204373.742081:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204373.742094:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559961472/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204373.742106:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204373.742111:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204373.742114:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204373.742114:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559961344:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204373.742123:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3504 00000100:00080000:0.0:1788204373.742126:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204373.742130:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204373.742136:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204373.742150:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559961600/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204373.742203:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204373.742271:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204373/real 1788204373] req@ffff93c0e9ce5c00 x1875064559961344/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204373.742293:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559961344/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204373.742305:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204373.742310:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204373.742314:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559961344:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204373.742344:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559961472:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204373.742365:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204373.742377:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204373/real 1788204373] req@ffff93c0e9ce6d80 x1875064559961472/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204373.742389:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559961472/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204373.742398:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204373.742401:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204373.742405:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559961472:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204373.742410:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559961600:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204373.742428:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204373.742437:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204373/real 1788204373] req@ffff93c0e9ce7b80 x1875064559961600/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204373.742448:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559961600/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204373.742467:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204373.742470:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204373.742474:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559961600:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204374.445994:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204375.470014:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204376.494009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204377.518005:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204378.349951:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204378.349966:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204378.350067:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204378.350104:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204378.350111:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204378.350160:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204378.542007:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204378.861930:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204378.861938:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204378.861940:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204378.861948:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3509 00000100:00080000:0.0:1788204378.861957:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204378.861964:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204378.861996:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559961728/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204378.862013:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204378.862015:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204378.862016:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204378.862019:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3509 00000100:00080000:0.0:1788204378.862021:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204378.862023:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204378.862030:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204378.862039:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559961728:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204378.862041:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559961856/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204378.862050:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204378.862052:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204378.862053:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204378.862055:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3509 00000100:00080000:0.0:1788204378.862057:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204378.862059:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204378.862062:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204378.862071:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559961984/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204378.862093:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204378.862129:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204378/real 1788204378] req@ffff93c0e9ce5f80 x1875064559961728/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204378.862142:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559961728/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204378.862149:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204378.862152:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204378.862155:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559961728:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204378.862174:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559961856:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204378.862195:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204378.862215:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204378/real 1788204378] req@ffff93c0e9ce7100 x1875064559961856/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204378.862227:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559961856/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204378.862232:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204378.862234:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204378.862236:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559961856:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204378.862240:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559961984:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204378.862255:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204378.862263:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204378/real 1788204378] req@ffff93c0e9ce4a80 x1875064559961984/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204378.862273:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559961984/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204378.862283:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204378.862286:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204378.862288:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559961984:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204379.565970:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204380.590040:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204381.614006:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204382.637987:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204383.470036:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204383.470056:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204383.470182:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204383.470211:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204383.470215:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204383.470246:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204383.661937:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204383.981990:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204383.982001:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204383.982007:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204383.982018:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3514 00000100:00080000:0.0:1788204383.982031:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204383.982040:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204383.982090:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559962112/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204383.982120:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204383.982123:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204383.982124:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204383.982128:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3514 00000100:00080000:0.0:1788204383.982131:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204383.982134:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204383.982144:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204383.982154:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559962240/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204383.982161:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204383.982164:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559962112:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204383.982165:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204383.982168:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204383.982175:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3514 00000100:00080000:0.0:1788204383.982177:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204383.982179:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204383.982183:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204383.982191:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559962368/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204383.982234:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204383.982297:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204383/real 1788204383] req@ffff93c0e9ce5180 x1875064559962112/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204383.982317:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559962112/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204383.982327:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204383.982331:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204383.982336:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559962112:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204383.982363:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559962240:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204383.982387:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204383.982401:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204383/real 1788204383] req@ffff93c0e9ce5500 x1875064559962240/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204383.982414:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559962240/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204383.982423:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204383.982426:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204383.982429:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559962240:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204383.982435:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559962368:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204383.982455:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204383.982466:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204383/real 1788204383] req@ffff93c0e9ce7800 x1875064559962368/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204383.982489:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559962368/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204383.982498:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204383.982501:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204383.982504:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559962368:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204384.686001:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204385.710024:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204386.733983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204387.758052:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204388.589989:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204388.590004:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204388.590124:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204388.590145:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204388.590148:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204388.590164:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204388.781994:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204389.101946:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204389.101961:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204389.101965:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204389.101978:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3519 00000100:00080000:0.0:1788204389.101991:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204389.102009:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204389.102052:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559962496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204389.102080:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204389.102084:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204389.102086:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204389.102091:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3519 00000100:00080000:0.0:1788204389.102095:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204389.102100:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204389.102112:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204389.102125:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559962624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204389.102127:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559962496:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204389.102136:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204389.102139:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204389.102141:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204389.102144:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3519 00000100:00080000:0.0:1788204389.102147:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204389.102150:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204389.102153:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204389.102162:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559962752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204389.102209:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204389.102246:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204389/real 1788204389] req@ffff93c0e9ce4e00 x1875064559962496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204389.102265:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559962496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204389.102283:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204389.102287:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204389.102292:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559962496:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204389.102322:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559962624:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204389.102345:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204389.102357:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204389/real 1788204389] req@ffff93c0e9ce5880 x1875064559962624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204389.102367:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559962624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204389.102375:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204389.102378:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204389.102381:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559962624:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204389.102387:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559962752:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204389.102407:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204389.102420:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204389/real 1788204389] req@ffff93c0e9ce6300 x1875064559962752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204389.102432:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559962752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204389.102454:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204389.102458:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204389.102461:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559962752:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204389.806074:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204390.830065:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204391.854058:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204392.878034:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204393.710003:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204393.710019:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204393.710146:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204393.710166:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204393.710169:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204393.710188:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204393.902011:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204394.221966:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204394.221976:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204394.221980:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204394.221991:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3525 00000100:00080000:2.0:1788204394.222002:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204394.222009:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204394.222057:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559962880/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204394.222086:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204394.222088:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204394.222090:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204394.222093:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3525 00000100:00080000:2.0:1788204394.222096:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204394.222098:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204394.222108:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204394.222119:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559963008/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204394.222125:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204394.222128:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204394.222129:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204394.222132:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3525 00000100:00080000:2.0:1788204394.222134:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204394.222136:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204394.222139:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204394.222147:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559963136/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204394.222155:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559962880:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788204394.222219:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204394.222271:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204394/real 1788204394] req@ffff93c0e62bb800 x1875064559962880/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204394.222292:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559962880/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204394.222303:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204394.222306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204394.222309:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559962880:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204394.222315:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559963008:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204394.222331:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204394.222340:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204394/real 1788204394] req@ffff93c0e62b9f80 x1875064559963008/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204394.222347:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559963008/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204394.222352:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204394.222366:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204394.222370:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559963008:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204394.222398:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559963136:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204394.222413:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204394.222419:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204394/real 1788204394] req@ffff93c0e62b9500 x1875064559963136/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204394.222427:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559963136/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204394.222437:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204394.222439:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204394.222441:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559963136:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204394.926027:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204395.949988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204396.974057:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204397.998030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204398.830039:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204398.830058:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204398.830183:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204398.830215:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204398.830219:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204398.830241:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204399.021953:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204399.341938:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204399.341949:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204399.341953:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204399.341964:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3530 00000100:00080000:2.0:1788204399.341976:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204399.341983:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204399.342028:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559963264/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204399.342053:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204399.342056:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204399.342057:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204399.342060:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3530 00000100:00080000:2.0:1788204399.342062:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204399.342065:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204399.342074:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204399.342082:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559963392/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204399.342089:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204399.342093:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204399.342094:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204399.342098:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559963264:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204399.342102:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3530 00000100:00080000:2.0:1788204399.342109:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204399.342113:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204399.342118:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204399.342131:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559963520/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204399.342176:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204399.342214:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204399/real 1788204399] req@ffff93c0e62b8a80 x1875064559963264/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204399.342228:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559963264/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204399.342237:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204399.342241:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204399.342244:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559963264:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204399.342265:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559963392:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204399.342303:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204399.342312:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204399/real 1788204399] req@ffff93c0e62b9c00 x1875064559963392/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204399.342320:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559963392/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204399.342330:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204399.342332:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204399.342334:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559963392:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204399.342337:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559963520:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204399.342353:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204399.342363:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204399/real 1788204399] req@ffff93c0e62bad80 x1875064559963520/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204399.342374:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559963520/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204399.342383:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204399.342386:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204399.342390:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559963520:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204400.046075:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204401.070025:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204402.093954:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204403.118056:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204403.950022:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204403.950041:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204403.950185:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204403.950243:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204403.950248:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204403.951614:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204404.141948:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204404.461978:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204404.461991:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204404.461997:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204404.462009:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3535 00000100:00080000:2.0:1788204404.462023:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204404.462031:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204404.462081:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559963648/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204404.462111:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204404.462115:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204404.462116:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204404.462119:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3535 00000100:00080000:2.0:1788204404.462122:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204404.462125:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204404.462142:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204404.462150:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559963648:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788204404.462155:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559963776/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204404.462164:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204404.462166:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204404.462167:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204404.462171:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3535 00000100:00080000:2.0:1788204404.462172:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204404.462174:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204404.462190:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204404.462199:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559963904/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204404.462239:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204404.462299:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204404/real 1788204404] req@ffff93c0e62bbb80 x1875064559963648/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204404.462320:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559963648/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204404.462330:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204404.462334:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204404.462339:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559963648:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204404.462368:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559963776:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204404.462391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204404.462417:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204404/real 1788204404] req@ffff93c0e62bb100 x1875064559963776/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204404.462430:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559963776/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204404.462437:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204404.462440:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204404.462443:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559963776:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204404.462449:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559963904:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204404.462470:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204404.462481:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204404/real 1788204404] req@ffff93c0e62b8700 x1875064559963904/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204404.462491:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559963904/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204404.462501:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204404.462504:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204404.462506:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559963904:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204405.166070:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204406.189946:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204407.214006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204408.238001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204409.070034:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204409.070049:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204409.070183:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204409.070208:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204409.070211:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204409.070231:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204409.262009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204409.581960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204409.581972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204409.581976:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204409.581987:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3540 00000100:00080000:1.0:1788204409.581997:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204409.582005:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204409.582042:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559964032/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204409.582073:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204409.582077:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204409.582080:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204409.582084:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3540 00000100:00080000:1.0:1788204409.582118:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204409.582124:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204409.582133:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204409.582136:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559964032:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204409.582149:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559964160/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204409.582162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204409.582166:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204409.582168:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204409.582172:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3540 00000100:00080000:1.0:1788204409.582180:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204409.582185:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204409.582190:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204409.582202:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559964288/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204409.582225:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204409.582262:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204409/real 1788204409] req@ffff93c1c5034a80 x1875064559964032/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204409.582276:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559964032/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204409.582283:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204409.582286:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204409.582288:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559964032:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204409.582305:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559964160:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204409.582320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204409.582331:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204409/real 1788204409] req@ffff93c1c5034e00 x1875064559964160/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204409.582342:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559964160/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204409.582349:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204409.582352:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204409.582356:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559964160:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204409.582361:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559964288:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204409.582380:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204409.582391:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204409/real 1788204409] req@ffff93c1c5037100 x1875064559964288/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204409.582398:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559964288/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204409.582405:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204409.582407:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204409.582409:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559964288:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204410.285952:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204411.310015:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204412.333941:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204413.358073:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204414.189978:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204414.189996:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204414.190106:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204414.190135:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204414.190140:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204414.190170:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204414.381981:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204414.701969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204414.701979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204414.701983:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204414.701991:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3545 00000100:00080000:3.0:1788204414.702016:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204414.702023:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204414.702056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559964416/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204414.702083:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204414.702086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204414.702088:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204414.702091:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3545 00000100:00080000:3.0:1788204414.702094:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204414.702098:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204414.702108:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204414.702120:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559964544/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204414.702131:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559964416:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788204414.702131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204414.702137:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204414.702140:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204414.702145:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3545 00000100:00080000:3.0:1788204414.702149:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204414.702153:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204414.702158:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788204414.702171:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559964672/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204414.702190:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204414.702230:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204414/real 1788204414] req@ffff93c1c4a4ea00 x1875064559964416/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204414.702242:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559964416/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204414.702249:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204414.702252:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204414.702255:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559964416:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204414.702272:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559964544:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204414.702285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204414.702292:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204414/real 1788204414] req@ffff93c1c4a4c000 x1875064559964544/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204414.702299:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559964544/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204414.702304:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204414.702306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204414.702308:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559964544:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204414.702311:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559964672:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204414.702329:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204414.702335:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204414/real 1788204414] req@ffff93c1c4a4fb80 x1875064559964672/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204414.702346:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559964672/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204414.702352:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204414.702354:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204414.702356:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559964672:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204415.405983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204416.430048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204417.454082:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204418.478024:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204419.309997:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204419.310015:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204419.310133:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204419.310169:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204419.310174:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204419.310202:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204419.502041:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204419.821978:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204419.821989:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204419.821994:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204419.822005:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3550 00000100:00080000:3.0:1788204419.822017:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204419.822025:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204419.822075:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559964800/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204419.822100:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204419.822104:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204419.822106:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204419.822111:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3550 00000100:00080000:3.0:1788204419.822116:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204419.822120:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204419.822131:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204419.822144:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559964928/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204419.822148:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559964800:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788204419.822153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204419.822156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204419.822158:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204419.822162:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3550 00000100:00080000:3.0:1788204419.822164:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204419.822168:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204419.822172:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788204419.822181:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559965056/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204419.822237:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204419.822292:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204419/real 1788204419] req@ffff93c1c4a4ed80 x1875064559964800/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204419.822307:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559964800/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204419.822315:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204419.822318:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204419.822321:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559964800:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204419.822343:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559964928:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204419.822359:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204419.822366:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204419/real 1788204419] req@ffff93c1c4a4dc00 x1875064559964928/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204419.822373:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559964928/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204419.822379:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204419.822381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204419.822383:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559964928:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204419.822386:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559965056:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204419.822396:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204419.822401:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204419/real 1788204419] req@ffff93c1c4a4e680 x1875064559965056/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204419.822408:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559965056/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204419.822415:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204419.822416:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204419.822418:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559965056:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204420.526067:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204421.550051:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204422.574045:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204423.598034:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204424.430026:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204424.430043:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204424.430178:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204424.430206:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204424.430211:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204424.430239:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204424.621987:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204424.941983:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204424.941991:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204424.941994:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204424.942008:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3555 00000100:00080000:2.0:1788204424.942019:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204424.942025:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204424.942061:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559965184/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204424.942087:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204424.942092:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204424.942094:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204424.942097:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3555 00000100:00080000:2.0:1788204424.942099:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204424.942102:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204424.942110:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204424.942128:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559965184:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788204424.942158:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559965312/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204424.942166:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204424.942169:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204424.942170:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204424.942173:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3555 00000100:00080000:2.0:1788204424.942177:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204424.942179:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204424.942182:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000800:00000100:0.0:1788204424.942187:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788204424.942191:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559965440/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204424.942248:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204424/real 1788204424] req@ffff93c0e62b8380 x1875064559965184/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204424.942267:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559965184/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204424.942283:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204424.942286:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204424.942289:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559965184:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204424.942307:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559965312:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204424.942324:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204424.942333:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204424/real 1788204424] req@ffff93c0e62b8000 x1875064559965312/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204424.942341:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559965312/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204424.942346:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204424.942347:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204424.942349:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559965312:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204424.942353:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559965440:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204424.942363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204424.942368:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204424/real 1788204424] req@ffff93c0e62ba680 x1875064559965440/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204424.942374:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559965440/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204424.942380:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204424.942382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204424.942383:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559965440:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204425.645952:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204426.670024:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204427.693961:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204428.717976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204429.550035:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204429.550052:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204429.550171:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204429.550192:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204429.550195:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204429.550211:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204429.742038:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204430.061959:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204430.061975:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204430.061980:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204430.061992:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3560 00000100:00080000:0.0:1788204430.062005:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204430.062014:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204430.062065:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559965568/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204430.062098:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204430.062101:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204430.062102:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204430.062106:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3560 00000100:00080000:0.0:1788204430.062109:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204430.062113:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204430.062123:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204430.062134:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559965696/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204430.062140:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559965568:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204430.062153:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204430.062156:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204430.062157:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204430.062161:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3560 00000100:00080000:0.0:1788204430.062164:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204430.062166:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204430.062170:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204430.062178:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559965824/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204430.062213:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204430.062272:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204430/real 1788204430] req@ffff93c0e9ce6300 x1875064559965568/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204430.062290:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559965568/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204430.062298:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204430.062301:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204430.062304:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559965568:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204430.062327:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559965696:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204430.062351:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204430.062366:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204430/real 1788204430] req@ffff93c0e9ce5880 x1875064559965696/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204430.062379:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559965696/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204430.062388:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204430.062391:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204430.062394:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559965696:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204430.062400:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559965824:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204430.062415:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204430.062423:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204430/real 1788204430] req@ffff93c0e9ce4e00 x1875064559965824/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204430.062430:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559965824/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204430.062437:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204430.062439:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204430.062441:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559965824:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204430.765978:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204431.790040:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204432.814020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204433.838057:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204434.670070:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204434.670085:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204434.670202:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204434.670240:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204434.670245:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204434.670270:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204434.861987:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204435.181975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204435.181982:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204435.181986:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204435.181999:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3566 00000100:00080000:2.0:1788204435.182008:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204435.182013:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204435.182049:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559965952/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204435.182075:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204435.182078:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204435.182080:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204435.182083:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3566 00000100:00080000:2.0:1788204435.182085:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204435.182088:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204435.182096:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204435.182106:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559966080/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204435.182112:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204435.182115:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204435.182116:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204435.182120:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3566 00000100:00080000:2.0:1788204435.182122:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788204435.182126:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559965952:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204435.182127:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204435.182131:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204435.182141:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559966208/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204435.182221:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559966080:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204435.182266:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559966208:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204435.886072:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204436.910034:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204437.934059:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788204438.254153:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788204438.254209:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788204438.254236:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204438.254286:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204438.254298:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:1.0:1788204438.254382:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204435/real 1788204438] req@ffff93c0e62ba680 x1875064559966208/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204438.254419:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559966208/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204438.254434:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204438.254440:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204438.254445:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559966208:192.168.201.156@tcp:8:kworker.0 00000100:00100000:1.0:1788204438.254454:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204435/real 1788204438] req@ffff93c0e62bb480 x1875064559965952/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204438.254467:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559965952/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204438.254475:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204438.254478:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204438.254482:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559965952:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204438.254488:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204435/real 1788204438] req@ffff93c0e62baa00 x1875064559966080/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204438.254500:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559966080/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204438.254507:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204438.254511:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204438.254514:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559966080:192.168.201.156@tcp:38:kworker.0 00010000:00010000:0.0:1788204438.958027:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204439.790049:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204439.790067:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204439.790201:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204439.790230:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204439.790235:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204439.790264:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204439.982380:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204440.301910:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204440.301925:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204440.301929:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204440.301939:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3571 00000100:00080000:2.0:1788204440.301950:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204440.301957:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204440.302006:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559966336/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204440.302034:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204440.302037:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204440.302038:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204440.302044:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3571 00000100:00080000:2.0:1788204440.302047:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204440.302051:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204440.302062:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204440.302076:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559966464/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204440.302087:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204440.302092:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1788204440.302093:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559966336:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204440.302094:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204440.302105:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3571 00000100:00080000:2.0:1788204440.302109:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204440.302113:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204440.302121:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204440.302146:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559966592/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204440.302172:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204440.302224:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204440/real 1788204440] req@ffff93c0e62b8000 x1875064559966336/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204440.302242:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559966336/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204440.302252:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204440.302256:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204440.302260:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559966336:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204440.302288:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559966464:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204440.302308:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204440.302323:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204440/real 1788204440] req@ffff93c0e62b8380 x1875064559966464/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204440.302333:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559966464/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204440.302340:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204440.302343:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204440.302346:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559966464:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204440.302352:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559966592:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204440.302371:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204440.302381:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204440/real 1788204440] req@ffff93c0e62b8700 x1875064559966592/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204440.302397:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559966592/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204440.302406:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204440.302409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204440.302413:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559966592:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204441.006051:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204442.029978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204443.053974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204444.078003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204444.910088:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204444.910104:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204444.910198:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204444.910220:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204444.910225:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204444.910251:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204445.102082:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204445.422000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204445.422015:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204445.422020:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204445.422034:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3576 00000100:00080000:0.0:1788204445.422047:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204445.422056:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204445.422107:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559966720/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204445.422140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204445.422144:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204445.422146:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204445.422151:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3576 00000100:00080000:0.0:1788204445.422155:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204445.422160:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204445.422173:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204445.422185:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559966720:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204445.422189:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559966848/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204445.422202:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204445.422206:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204445.422207:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204445.422212:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3576 00000100:00080000:0.0:1788204445.422216:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204445.422220:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204445.422226:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204445.422240:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559966976/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204445.422265:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204445.422319:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204445/real 1788204445] req@ffff93c0e9ce7800 x1875064559966720/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204445.422337:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559966720/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204445.422349:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204445.422353:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204445.422357:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559966720:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204445.422384:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559966848:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204445.422410:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204445.422423:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204445/real 1788204445] req@ffff93c0e9ce5500 x1875064559966848/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204445.422435:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559966848/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204445.422450:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204445.422454:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204445.422457:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559966848:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204445.422463:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559966976:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204445.422511:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204445.422523:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204445/real 1788204445] req@ffff93c0e9ce5180 x1875064559966976/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204445.422535:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559966976/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204445.422546:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204445.422549:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204445.422552:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559966976:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204446.126012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204447.150054:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204448.174037:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204449.198019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204450.029989:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204450.030003:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204450.030087:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204450.030107:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204450.030109:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204450.030125:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204450.222035:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204450.541995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204450.542009:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204450.542014:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204450.542027:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3581 00000100:00080000:0.0:1788204450.542041:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204450.542050:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204450.542102:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559967104/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204450.542139:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204450.542143:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204450.542145:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204450.542150:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3581 00000100:00080000:0.0:1788204450.542154:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204450.542159:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204450.542171:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204450.542185:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559967232/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204450.542195:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559967104:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204450.542199:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204450.542205:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204450.542206:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204450.542212:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3581 00000100:00080000:0.0:1788204450.542217:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204450.542220:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204450.542226:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204450.542238:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559967360/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204450.542265:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204450.542321:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204450/real 1788204450] req@ffff93c0e9ce4a80 x1875064559967104/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204450.542348:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559967104/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204450.542360:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204450.542364:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204450.542368:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559967104:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204450.542423:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559967232:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204450.542442:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204450.542451:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204450/real 1788204450] req@ffff93c0e9ce7100 x1875064559967232/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204450.542459:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559967232/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204450.542464:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204450.542466:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204450.542468:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559967232:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204450.542474:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559967360:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204450.542492:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204450.542503:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204450/real 1788204450] req@ffff93c0e9ce5f80 x1875064559967360/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204450.542515:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559967360/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204450.542531:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204450.542534:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204450.542537:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559967360:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204451.246020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204452.269993:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204453.293979:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204454.318005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204455.149995:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204455.150007:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204455.150082:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204455.150098:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204455.150100:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204455.150115:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204455.341931:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204455.661925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204455.661935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204455.661939:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204455.661949:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3586 00000100:00080000:0.0:1788204455.661960:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204455.661967:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204455.662005:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559967488/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204455.662046:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204455.662049:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204455.662051:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204455.662058:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3586 00000100:00080000:0.0:1788204455.662062:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204455.662066:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204455.662075:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204455.662082:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559967488:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204455.662091:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559967616/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204455.662102:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204455.662105:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204455.662106:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204455.662110:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3586 00000100:00080000:0.0:1788204455.662113:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204455.662117:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204455.662121:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204455.662131:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559967744/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204455.662149:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204455.662202:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204455/real 1788204455] req@ffff93c0e9ce7b80 x1875064559967488/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204455.662229:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559967488/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204455.662239:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204455.662243:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204455.662246:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559967488:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204455.662272:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559967616:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204455.662313:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204455.662325:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204455/real 1788204455] req@ffff93c0e9ce6d80 x1875064559967616/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204455.662336:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559967616/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204455.662343:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204455.662346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204455.662348:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559967616:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204455.662354:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559967744:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204455.662371:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204455.662380:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204455/real 1788204455] req@ffff93c0e9ce5c00 x1875064559967744/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204455.662390:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559967744/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204455.662398:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204455.662402:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204455.662405:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559967744:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204456.366037:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204457.390013:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204458.414026:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204459.437998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204460.270031:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204460.270048:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204460.270148:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204460.270168:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204460.270170:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204460.270185:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204460.461982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204460.781963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204460.781974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204460.781979:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204460.781988:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3591 00000100:00080000:0.0:1788204460.782000:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204460.782007:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204460.782058:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559967872/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204460.782081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204460.782083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204460.782085:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204460.782088:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3591 00000100:00080000:0.0:1788204460.782090:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204460.782092:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204460.782102:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204460.782110:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559968000/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204460.782118:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204460.782120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204460.782121:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204460.782125:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559967872:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204460.782127:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3591 00000100:00080000:0.0:1788204460.782129:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204460.782132:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204460.782137:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204460.782145:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559968128/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204460.782222:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204460.782268:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204460/real 1788204460] req@ffff93c0e9ce4000 x1875064559967872/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204460.782283:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559967872/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204460.782294:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204460.782298:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204460.782302:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559967872:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204460.782329:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559968000:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204460.782349:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204460.782358:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204460/real 1788204460] req@ffff93c0e9ce6a00 x1875064559968000/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204460.782367:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559968000/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204460.782373:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204460.782374:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204460.782376:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559968000:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204460.782380:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559968128:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204460.782391:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204460.782396:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204460/real 1788204460] req@ffff93c0e9ce6680 x1875064559968128/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204460.782411:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559968128/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204460.782417:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204460.782420:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204460.782422:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559968128:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204461.486005:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204462.510027:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204463.534007:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204464.558053:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204465.390011:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204465.390033:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204465.390159:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204465.390188:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204465.390193:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204465.390215:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204465.582013:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204465.902011:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204465.902022:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204465.902026:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204465.902036:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3596 00000100:00080000:0.0:1788204465.902046:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204465.902053:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204465.902094:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559968256/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204465.902129:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204465.902131:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204465.902133:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204465.902136:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3596 00000100:00080000:0.0:1788204465.902139:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204465.902142:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204465.902152:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204465.902162:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559968384/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204465.902170:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204465.902172:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204465.902174:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204465.902178:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3596 00000100:00080000:0.0:1788204465.902181:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788204465.902182:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559968256:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204465.902184:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204465.902191:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204465.902200:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559968512/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204465.902256:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204465.902308:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204465/real 1788204465] req@ffff93c0e9ce4380 x1875064559968256/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204465.902328:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559968256/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204465.902346:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204465.902351:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204465.902355:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559968256:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204465.902381:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559968384:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204465.902416:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204465.902429:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204465/real 1788204465] req@ffff93c0e9ce4700 x1875064559968384/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204465.902441:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559968384/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204465.902450:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204465.902453:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204465.902456:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559968384:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204465.902463:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559968512:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204465.902480:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204465.902489:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204465/real 1788204465] req@ffff93c0e9ce7480 x1875064559968512/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204465.902500:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559968512/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204465.902511:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204465.902514:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204465.902517:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559968512:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204466.606139:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204467.629974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204468.654092:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204469.677990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204470.509988:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204470.510008:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204470.510165:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204470.510193:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204470.510197:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204470.510218:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204470.702063:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204471.021876:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204471.021883:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204471.021886:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204471.021893:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3601 00000100:00080000:0.0:1788204471.021901:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204471.021906:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204471.021935:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559968640/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204471.021953:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204471.021955:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204471.021956:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204471.021959:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3601 00000100:00080000:0.0:1788204471.021961:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204471.021963:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204471.021970:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204471.021978:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559968768/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204471.021984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204471.021986:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204471.021988:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204471.021992:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559968640:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204471.021993:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3601 00000100:00080000:0.0:1788204471.021997:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204471.022002:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204471.022006:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204471.022012:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559968896/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204471.022071:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204471.022108:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204471/real 1788204471] req@ffff93c0e9ce7480 x1875064559968640/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204471.022122:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559968640/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204471.022129:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204471.022132:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204471.022135:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559968640:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204471.022154:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559968768:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204471.022175:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204471.022187:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204471/real 1788204471] req@ffff93c0e9ce4700 x1875064559968768/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204471.022198:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559968768/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204471.022203:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204471.022205:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204471.022207:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559968768:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204471.022211:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559968896:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204471.022224:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204471.022230:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204471/real 1788204471] req@ffff93c0e9ce4380 x1875064559968896/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204471.022237:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559968896/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204471.022243:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204471.022245:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204471.022247:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559968896:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204471.725953:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204472.750036:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204473.773945:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204474.798059:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204475.630005:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204475.630016:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204475.630101:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204475.630116:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204475.630119:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204475.630134:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204475.822031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204476.141964:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204476.141972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204476.141975:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204476.141982:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3607 00000100:00080000:2.0:1788204476.141991:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204476.141996:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204476.142029:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559969024/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204476.142057:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204476.142061:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204476.142062:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204476.142066:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3607 00000100:00080000:2.0:1788204476.142070:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204476.142073:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204476.142082:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204476.142097:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559969152/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204476.142108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204476.142111:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204476.142114:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204476.142117:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559969024:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204476.142121:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3607 00000100:00080000:2.0:1788204476.142127:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204476.142131:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204476.142136:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204476.142147:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559969280/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204476.142190:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204476.142229:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204476/real 1788204476] req@ffff93c0e62bb100 x1875064559969024/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204476.142243:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559969024/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204476.142250:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204476.142253:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204476.142257:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559969024:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204476.142287:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559969152:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204476.142309:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204476.142319:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204476/real 1788204476] req@ffff93c0e62bbb80 x1875064559969152/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204476.142326:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559969152/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204476.142335:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204476.142337:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204476.142339:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559969152:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204476.142343:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559969280:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204476.142354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204476.142360:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204476/real 1788204476] req@ffff93c0e62bad80 x1875064559969280/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204476.142366:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559969280/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204476.142372:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204476.142373:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204476.142375:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559969280:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204476.846041:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204477.870025:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204478.894046:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204479.918037:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204480.750066:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204480.750081:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204480.750174:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204480.750198:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204480.750202:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204480.750227:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204480.942025:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204481.261988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204481.261996:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204481.262000:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204481.262007:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3612 00000100:00080000:0.0:1788204481.262018:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204481.262023:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204481.262055:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559969408/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204481.262076:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204481.262078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204481.262080:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204481.262082:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3612 00000100:00080000:0.0:1788204481.262085:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204481.262087:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204481.262094:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204481.262101:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559969536/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204481.262108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204481.262112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204481.262113:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204481.262116:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3612 00000100:00080000:0.0:1788204481.262118:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204481.262120:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204481.262123:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204481.262131:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559969664/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204481.262133:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559969408:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788204481.262201:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204481.262241:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204481/real 1788204481] req@ffff93c0e9ce6680 x1875064559969408/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204481.262258:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559969408/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204481.262267:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204481.262271:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204481.262275:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559969408:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204481.262283:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559969536:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204481.262304:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204481.262316:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204481/real 1788204481] req@ffff93c0e9ce6a00 x1875064559969536/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204481.262327:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559969536/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204481.262334:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204481.262337:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204481.262340:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559969536:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204481.262363:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559969664:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204481.262381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204481.262392:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204481/real 1788204481] req@ffff93c0e9ce4000 x1875064559969664/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204481.262402:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559969664/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204481.262410:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204481.262413:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204481.262416:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559969664:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204481.965957:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204482.990048:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204484.014019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204485.038017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204485.870026:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204485.870039:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204485.870133:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204485.870156:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204485.870159:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204485.870181:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204486.062073:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204486.381961:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204486.381971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204486.381976:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204486.381986:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3617 00000100:00080000:0.0:1788204486.381997:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204486.382005:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204486.382042:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559969792/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204486.382068:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204486.382072:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204486.382075:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204486.382080:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3617 00000100:00080000:0.0:1788204486.382083:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204486.382087:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204486.382097:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204486.382110:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559969920/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204486.382123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204486.382123:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559969792:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204486.382131:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204486.382134:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204486.382139:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3617 00000100:00080000:0.0:1788204486.382143:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204486.382147:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204486.382153:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204486.382166:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559970048/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204486.382193:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204486.382239:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204486/real 1788204486] req@ffff93c0e9ce5c00 x1875064559969792/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204486.382258:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559969792/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204486.382269:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204486.382272:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204486.382276:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559969792:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204486.382298:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559969920:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204486.382322:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204486.382335:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204486/real 1788204486] req@ffff93c0e9ce6d80 x1875064559969920/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204486.382347:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559969920/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204486.382355:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204486.382358:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204486.382362:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559969920:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204486.382368:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559970048:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204486.382389:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204486.382400:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204486/real 1788204486] req@ffff93c0e9ce7b80 x1875064559970048/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204486.382411:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559970048/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204486.382426:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204486.382429:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204486.382432:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559970048:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204487.085992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204488.110052:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204489.134024:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204490.158070:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204490.990080:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204490.990097:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204490.990211:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204490.990232:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204490.990235:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204490.990251:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204491.181984:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204491.501988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204491.502003:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204491.502009:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204491.502021:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3622 00000100:00080000:0.0:1788204491.502034:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204491.502043:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204491.502093:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559970176/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204491.502125:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204491.502130:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204491.502132:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204491.502136:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3622 00000100:00080000:0.0:1788204491.502140:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204491.502149:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204491.502160:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204491.502175:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559970176:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204491.502176:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559970304/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204491.502192:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204491.502196:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204491.502198:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204491.502204:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3622 00000100:00080000:0.0:1788204491.502208:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204491.502212:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204491.502217:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204491.502228:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559970432/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204491.502245:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204491.502291:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204491/real 1788204491] req@ffff93c0e9ce5f80 x1875064559970176/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204491.502305:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559970176/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204491.502313:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204491.502316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204491.502319:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559970176:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204491.502346:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559970304:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204491.502361:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204491.502369:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204491/real 1788204491] req@ffff93c0e9ce7100 x1875064559970304/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204491.502377:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559970304/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204491.502382:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204491.502384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204491.502386:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559970304:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204491.502390:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559970432:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204491.502407:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204491.502417:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204491/real 1788204491] req@ffff93c0e9ce4a80 x1875064559970432/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204491.502428:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559970432/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204491.502438:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204491.502442:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204491.502445:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559970432:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204492.206022:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204493.230054:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204494.254037:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204495.278086:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204496.110076:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204496.110093:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204496.110220:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204496.110242:0:8:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204496.110245:0:8:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204496.110263:0:8:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204496.302040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204496.622039:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204496.622054:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204496.622060:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204496.622076:0:8:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3627 00000100:00080000:2.0:1788204496.622090:0:8:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204496.622098:0:8:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204496.622148:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559970560/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204496.622176:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204496.622179:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204496.622181:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204496.622185:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3627 00000100:00080000:2.0:1788204496.622188:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204496.622192:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204496.622202:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204496.622213:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559970688/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204496.622222:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204496.622225:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204496.622227:0:8:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204496.622232:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3627 00000100:00080000:2.0:1788204496.622236:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204496.622240:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204496.622250:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204496.622261:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559970816/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204496.622277:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559970560:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788204496.622360:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204496.622427:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204496/real 1788204496] req@ffff93c0e62b9c00 x1875064559970560/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204496.622448:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559970560/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204496.622459:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204496.622463:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204496.622466:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559970560:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204496.622474:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559970688:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204496.622497:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204496.622509:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204496/real 1788204496] req@ffff93c0e62b8a80 x1875064559970688/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204496.622521:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559970688/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204496.622529:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204496.622532:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204496.622535:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559970688:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204496.622561:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559970816:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204496.622596:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204496.622610:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204496/real 1788204496] req@ffff93c0e62b9500 x1875064559970816/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204496.622622:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559970816/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204496.622631:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204496.622635:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204496.622638:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559970816:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204497.326047:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204498.350031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204499.374026:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204500.398055:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204501.230011:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204501.230026:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204501.230132:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204501.230160:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204501.230164:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204501.230194:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204501.422003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204501.741942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204501.741950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204501.741953:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204501.741961:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3632 00000100:00080000:2.0:1788204501.741971:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204501.741980:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204501.742015:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559970944/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204501.742038:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204501.742041:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204501.742042:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204501.742045:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3632 00000100:00080000:2.0:1788204501.742047:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204501.742050:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204501.742057:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204501.742066:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559971072/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204501.742073:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204501.742076:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204501.742077:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204501.742079:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3632 00000100:00080000:2.0:1788204501.742081:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204501.742083:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204501.742087:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204501.742090:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559970944:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788204501.742097:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559971200/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204501.742174:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559971072:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204501.742202:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559971200:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204502.446021:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204503.470057:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204504.493986:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204504.814126:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:2.0:1788204504.814172:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788204504.814188:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:1.0:1788204504.814317:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204501/real 1788204504] req@ffff93c0e62b9f80 x1875064559970944/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204526 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204504.814350:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559970944/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204526 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204504.814361:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204504.814365:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204504.814369:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559970944:192.168.201.156@tcp:250:kworker.0 00000800:00000100:2.0:1788204504.814466:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:1.0:1788204504.814539:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204501/real 1788204504] req@ffff93c0e62bb800 x1875064559971072/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204526 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204504.814553:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559971072/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204526 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204504.814560:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204504.814563:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204504.814566:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559971072:192.168.201.156@tcp:38:kworker.0 00000800:00000100:2.0:1788204504.814611:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:1.0:1788204504.814919:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204501/real 1788204504] req@ffff93c0e62b9500 x1875064559971200/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204526 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204504.814936:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559971200/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204526 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204504.814947:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204504.814951:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204504.814954:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559971200:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204505.517992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204506.349988:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204506.350001:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204506.350110:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204506.350147:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204506.350151:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204506.350177:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204506.542044:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204506.862031:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204506.862042:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204506.862045:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204506.862055:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3637 00000100:00080000:2.0:1788204506.862065:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204506.862072:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204506.862102:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559971328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204506.862136:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204506.862139:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204506.862141:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204506.862145:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3637 00000100:00080000:2.0:1788204506.862149:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204506.862152:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204506.862161:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204506.862172:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559971456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204506.862179:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204506.862182:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204506.862184:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204506.862187:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3637 00000100:00080000:2.0:1788204506.862191:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204506.862194:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204506.862198:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204506.862208:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559971584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204506.862317:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559971328:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788204506.862395:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204506.862450:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204506/real 1788204506] req@ffff93c0e62b9500 x1875064559971328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204506.862473:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559971328/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204506.862485:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204506.862490:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204506.862493:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559971328:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204506.862500:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559971456:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204506.862522:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204506.862535:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204506/real 1788204506] req@ffff93c0e62bb800 x1875064559971456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204506.862547:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559971456/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204506.862555:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204506.862558:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204506.862562:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559971456:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204506.862568:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559971584:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204506.862588:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204506.862600:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204506/real 1788204506] req@ffff93c0e62b9f80 x1875064559971584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204506.862619:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559971584/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204506.862629:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204506.862632:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204506.862635:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559971584:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204507.566005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204508.589976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204509.614132:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204510.638040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204511.470048:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204511.470068:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204511.470196:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204511.470233:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204511.470238:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204511.470266:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204511.662051:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204511.981982:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204511.981996:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204511.982001:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204511.982014:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3642 00000100:00080000:2.0:1788204511.982028:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204511.982037:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204511.982097:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559971712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204511.982128:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204511.982133:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204511.982135:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204511.982140:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3642 00000100:00080000:2.0:1788204511.982144:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204511.982148:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204511.982160:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204511.982176:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559971840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204511.982187:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204511.982191:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204511.982193:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204511.982198:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3642 00000100:00080000:2.0:1788204511.982203:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204511.982207:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204511.982213:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204511.982224:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559971712:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788204511.982230:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559971968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204511.982353:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204511.982412:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204511/real 1788204511] req@ffff93c0e62b8a80 x1875064559971712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204511.982432:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559971712/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204511.982443:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204511.982447:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204511.982451:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559971712:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204511.982459:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559971840:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204511.982482:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204511.982494:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204511/real 1788204511] req@ffff93c0e62b9c00 x1875064559971840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204511.982506:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559971840/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204511.982520:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204511.982523:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204511.982527:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559971840:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204511.982556:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559971968:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204511.982576:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204511.982588:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204511/real 1788204511] req@ffff93c0e62bad80 x1875064559971968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204511.982598:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559971968/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204511.982607:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204511.982611:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204511.982614:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559971968:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204512.686012:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204513.710022:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204514.734026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204515.758050:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204516.590013:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204516.590033:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204516.590171:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204516.590200:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204516.590204:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204516.590270:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204516.782015:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204517.102008:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204517.102022:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204517.102027:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204517.102038:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3647 00000100:00080000:2.0:1788204517.102052:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204517.102061:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204517.102112:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559972096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204517.102140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204517.102144:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204517.102146:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204517.102150:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3647 00000100:00080000:2.0:1788204517.102154:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204517.102165:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204517.102178:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204517.102191:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559972224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204517.102196:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559972096:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204517.102213:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204517.102217:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204517.102219:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204517.102237:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3647 00000100:00080000:2.0:1788204517.102241:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204517.102245:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204517.102251:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204517.102262:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559972352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204517.102304:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204517.102343:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204517/real 1788204517] req@ffff93c0e62bbb80 x1875064559972096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204517.102357:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559972096/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204517.102365:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204517.102368:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204517.102371:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559972096:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204517.102394:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559972224:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204517.102416:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204517.102428:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204517/real 1788204517] req@ffff93c0e62bb100 x1875064559972224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204517.102440:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559972224/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204517.102447:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204517.102450:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204517.102453:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559972224:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204517.102459:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559972352:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204517.102487:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204517.102497:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204517/real 1788204517] req@ffff93c0e62b8700 x1875064559972352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204517.102504:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559972352/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204517.102512:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204517.102516:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204517.102519:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559972352:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204517.806076:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204518.830055:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204519.854057:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204520.877994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204521.710043:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204521.710058:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204521.710185:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204521.710211:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204521.710215:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204521.710242:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204521.901995:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204522.222004:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204522.222014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204522.222018:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204522.222028:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3653 00000100:00080000:0.0:1788204522.222041:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204522.222048:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204522.222087:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559972480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204522.222116:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204522.222120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204522.222122:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204522.222126:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3653 00000100:00080000:0.0:1788204522.222130:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204522.222148:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204522.222158:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204522.222180:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559972608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204522.222199:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204522.222199:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559972480:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204522.222208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204522.222211:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204522.222216:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3653 00000100:00080000:0.0:1788204522.222220:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204522.222224:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204522.222229:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204522.222245:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559972736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204522.222280:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204522.222336:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204522/real 1788204522] req@ffff93c0e9ce5180 x1875064559972480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204522.222358:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559972480/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204522.222367:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204522.222371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204522.222375:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559972480:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204522.222407:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559972608:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204522.222431:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204522.222444:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204522/real 1788204522] req@ffff93c0e9ce5500 x1875064559972608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204522.222455:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559972608/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204522.222463:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204522.222466:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204522.222469:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559972608:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204522.222476:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559972736:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204522.222494:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204522.222504:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204522/real 1788204522] req@ffff93c0e9ce7800 x1875064559972736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204522.222514:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559972736/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204522.222529:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204522.222532:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204522.222536:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559972736:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204522.925981:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204523.950056:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204524.974024:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204525.998045:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204526.830002:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204526.830013:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204526.830131:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204526.830150:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204526.830153:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204526.830172:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204527.021955:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204527.341976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204527.341986:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204527.341989:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204527.341997:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3658 00000100:00080000:2.0:1788204527.342008:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204527.342014:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204527.342068:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559972864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204527.342099:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204527.342102:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204527.342103:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204527.342107:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3658 00000100:00080000:2.0:1788204527.342110:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204527.342112:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204527.342121:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204527.342131:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559972992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204527.342138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204527.342140:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204527.342141:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204527.342143:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3658 00000100:00080000:2.0:1788204527.342145:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204527.342148:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204527.342152:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204527.342160:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559973120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204527.342255:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559972864:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788204527.342311:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204527.342344:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204527/real 1788204527] req@ffff93c0e62b8380 x1875064559972864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204527.342357:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559972864/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204527.342364:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204527.342366:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204527.342368:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559972864:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204527.342372:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559972992:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204527.342384:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204527.342390:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204527/real 1788204527] req@ffff93c0e62b8000 x1875064559972992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204527.342400:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559972992/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204527.342407:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204527.342410:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204527.342414:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559972992:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204527.342420:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559973120:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204527.342438:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204527.342447:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204527/real 1788204527] req@ffff93c0e62baa00 x1875064559973120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204527.342458:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559973120/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204527.342466:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204527.342469:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204527.342473:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559973120:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204528.045990:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204529.070068:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204530.094010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204531.118023:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204531.950017:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204531.950031:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204531.950135:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204531.950159:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204531.950164:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204531.950190:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204532.142040:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204532.461921:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204532.461932:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204532.461936:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204532.461951:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3663 00000100:00080000:0.0:1788204532.461973:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204532.461981:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204532.462016:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559973248/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204532.462043:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204532.462047:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204532.462049:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204532.462053:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3663 00000100:00080000:0.0:1788204532.462057:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204532.462061:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204532.462072:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204532.462085:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559973376/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204532.462097:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559973248:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204532.462100:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204532.462105:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204532.462107:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204532.462112:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3663 00000100:00080000:0.0:1788204532.462116:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204532.462120:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204532.462126:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204532.462139:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559973504/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204532.462156:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204532.462191:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204532/real 1788204532] req@ffff93c0e9ce4e00 x1875064559973248/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204532.462205:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559973248/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204532.462213:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204532.462215:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204532.462218:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559973248:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204532.462236:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559973376:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204532.462250:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204532.462257:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204532/real 1788204532] req@ffff93c0e9ce5880 x1875064559973376/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204532.462265:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559973376/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204532.462270:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204532.462272:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204532.462274:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559973376:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204532.462277:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559973504:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204532.462286:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204532.462291:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204532/real 1788204532] req@ffff93c0e9ce6300 x1875064559973504/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204532.462298:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559973504/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204532.462303:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204532.462305:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204532.462306:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559973504:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204533.166025:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204534.190015:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204535.214070:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204536.238018:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204537.070054:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204537.070070:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204537.070165:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204537.070185:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204537.070188:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204537.070205:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788204537.263073:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204537.581961:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204537.581971:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204537.581975:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204537.581985:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3668 00000100:00080000:2.0:1788204537.581998:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204537.582005:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204537.582050:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559973632/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204537.582080:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204537.582084:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204537.582085:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204537.582090:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3668 00000100:00080000:2.0:1788204537.582093:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204537.582097:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204537.582109:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204537.582123:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559973760/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204537.582135:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559973632:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204537.582135:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204537.582143:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204537.582146:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204537.582151:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3668 00000100:00080000:2.0:1788204537.582155:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204537.582159:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204537.582164:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204537.582179:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559973888/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204537.582227:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204537.582274:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204537/real 1788204537] req@ffff93c0e62bb480 x1875064559973632/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204537.582290:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559973632/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204537.582301:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204537.582306:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204537.582310:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559973632:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204537.582340:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559973760:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204537.582358:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204537.582368:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204537/real 1788204537] req@ffff93c0e62ba680 x1875064559973760/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204537.582376:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559973760/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204537.582382:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204537.582384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204537.582386:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559973760:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204537.582390:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559973888:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204537.582401:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204537.582407:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204537/real 1788204537] req@ffff93c0e62baa00 x1875064559973888/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204537.582414:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559973888/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204537.582420:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204537.582422:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204537.582424:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559973888:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204538.286045:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788204539.310001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204540.333958:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788204541.358076:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204542.190044:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204542.190061:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204542.190197:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204542.190224:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204542.190229:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204542.190257:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204542.382006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204542.701959:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204542.701970:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204542.701975:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204542.701984:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3673 00000100:00080000:2.0:1788204542.701996:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204542.702003:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204542.702043:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559974016/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204542.702067:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204542.702069:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204542.702071:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204542.702073:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3673 00000100:00080000:2.0:1788204542.702075:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204542.702078:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204542.702087:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204542.702096:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559974144/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204542.702110:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204542.702112:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559974016:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204542.702116:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204542.702120:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204542.702126:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3673 00000100:00080000:2.0:1788204542.702129:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204542.702133:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204542.702140:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204542.702162:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559974272/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204542.702181:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204542.702231:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204542/real 1788204542] req@ffff93c0e62b8000 x1875064559974016/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204542.702252:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559974016/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204542.702264:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204542.702268:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204542.702273:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559974016:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204542.702299:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559974144:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204542.702320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204542.702329:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204542/real 1788204542] req@ffff93c0e62b8380 x1875064559974144/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204542.702336:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559974144/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204542.702342:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204542.702344:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204542.702346:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559974144:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204542.702350:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559974272:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204542.702372:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204542.702379:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204542/real 1788204542] req@ffff93c0e62b8700 x1875064559974272/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204542.702385:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559974272/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204542.702392:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204542.702394:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204542.702396:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559974272:192.168.201.156@tcp:8:kworker.0 00010000:00010000:1.0:1788204543.406034:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204544.429991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788204545.454032:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204546.478047:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204547.310033:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204547.310052:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204547.310167:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204547.310184:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204547.310187:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204547.310202:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788204547.501976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204547.821974:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204547.821988:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204547.821993:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204547.822011:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3678 00000100:00080000:2.0:1788204547.822026:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204547.822035:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204547.822087:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559974400/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204547.822116:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204547.822119:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204547.822120:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204547.822123:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3678 00000100:00080000:2.0:1788204547.822126:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204547.822128:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204547.822141:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204547.822154:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559974528/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204547.822165:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204547.822169:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204547.822173:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204547.822174:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559974400:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204547.822180:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3678 00000100:00080000:2.0:1788204547.822184:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204547.822187:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204547.822191:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204547.822201:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559974656/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204547.822257:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204547.822306:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204547/real 1788204547] req@ffff93c0e62bb100 x1875064559974400/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204547.822324:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559974400/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204547.822334:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204547.822338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204547.822342:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559974400:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204547.822368:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559974528:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204547.822389:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204547.822401:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204547/real 1788204547] req@ffff93c0e62bbb80 x1875064559974528/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204547.822414:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559974528/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204547.822422:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204547.822425:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204547.822428:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559974528:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204547.822434:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559974656:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204547.822453:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204547.822463:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204547/real 1788204547] req@ffff93c0e62bad80 x1875064559974656/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204547.822473:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559974656/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204547.822483:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204547.822486:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204547.822489:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559974656:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204548.526084:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788204549.550047:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204550.574013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788204551.598019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204552.430011:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204552.430026:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204552.430126:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204552.430150:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204552.430154:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204552.430178:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204552.622068:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204552.941951:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204552.941961:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204552.941965:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204552.941975:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3683 00000100:00080000:2.0:1788204552.941987:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204552.941994:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204552.942029:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559974784/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204552.942057:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204552.942061:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204552.942063:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204552.942066:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3683 00000100:00080000:2.0:1788204552.942070:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204552.942074:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204552.942083:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204552.942095:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559974912/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204552.942105:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204552.942110:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1788204552.942111:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559974784:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204552.942112:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204552.942120:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3683 00000100:00080000:2.0:1788204552.942124:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204552.942127:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204552.942132:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204552.942141:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559975040/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204552.942180:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204552.942219:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204552/real 1788204552] req@ffff93c0e62b9c00 x1875064559974784/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204552.942241:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559974784/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204552.942250:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204552.942254:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204552.942258:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559974784:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204552.942280:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559974912:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204552.942300:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204552.942313:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204552/real 1788204552] req@ffff93c0e62b8a80 x1875064559974912/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204552.942325:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559974912/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204552.942332:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204552.942335:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204552.942339:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559974912:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204552.942345:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559975040:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204552.942364:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204552.942376:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204552/real 1788204552] req@ffff93c0e62b9f80 x1875064559975040/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204552.942387:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559975040/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204552.942402:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204552.942406:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204552.942409:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559975040:192.168.201.156@tcp:8:kworker.0 00010000:00010000:1.0:1788204553.646010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204554.670008:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788204555.693986:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204556.717971:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204557.550067:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204557.550088:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204557.550208:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204557.550249:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204557.550254:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204557.550276:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788204557.742014:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204558.061988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204558.061996:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204558.062000:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204558.062010:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3688 00000100:00080000:1.0:1788204558.062020:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204558.062026:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204558.062063:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559975168/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204558.062091:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204558.062094:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204558.062096:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204558.062107:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3688 00000100:00080000:1.0:1788204558.062112:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204558.062116:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204558.062125:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204558.062139:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559975168:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204558.062140:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559975296/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204558.062154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204558.062157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204558.062159:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204558.062163:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3688 00000100:00080000:1.0:1788204558.062167:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204558.062171:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204558.062176:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204558.062188:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559975424/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204558.062203:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204558.062237:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204558/real 1788204558] req@ffff93c1c5035c00 x1875064559975168/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204558.062249:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559975168/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204558.062256:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204558.062259:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204558.062261:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559975168:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204558.062277:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559975296:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204558.062291:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204558.062298:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204558/real 1788204558] req@ffff93c1c5037800 x1875064559975296/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204558.062305:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559975296/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204558.062310:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204558.062312:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204558.062314:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559975296:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204558.062317:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559975424:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204558.062327:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204558.062331:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204558/real 1788204558] req@ffff93c1c5036680 x1875064559975424/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204558.062338:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559975424/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204558.062344:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204558.062346:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204558.062348:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559975424:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204558.766000:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204559.790025:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204560.814049:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204561.837962:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204562.670030:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204562.670046:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204562.670177:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204562.670204:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204562.670211:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204562.670231:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204562.862020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204563.181975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204563.181990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204563.181994:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204563.182004:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3694 00000100:00080000:1.0:1788204563.182016:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204563.182022:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204563.182064:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559975552/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204563.182090:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204563.182092:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204563.182093:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204563.182096:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3694 00000100:00080000:1.0:1788204563.182098:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204563.182100:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204563.182109:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204563.182117:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559975680/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204563.182124:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204563.182127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204563.182129:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204563.182131:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3694 00000100:00080000:1.0:1788204563.182133:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204563.182135:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204563.182137:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204563.182143:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559975808/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204563.182157:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559975552:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788204563.182239:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204563.182280:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204563/real 1788204563] req@ffff93c1c5036680 x1875064559975552/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204563.182294:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559975552/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204563.182302:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204563.182304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204563.182307:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559975552:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204563.182312:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559975680:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204563.182328:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204563.182335:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204563/real 1788204563] req@ffff93c1c5037800 x1875064559975680/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204563.182342:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559975680/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204563.182347:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204563.182349:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204563.182351:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559975680:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204563.182370:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559975808:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204563.182382:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204563.182388:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204563/real 1788204563] req@ffff93c1c5035c00 x1875064559975808/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204563.182395:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559975808/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204563.182401:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204563.182403:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204563.182404:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559975808:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204563.886044:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204564.909986:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204565.934032:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204566.958047:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204567.789995:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204567.790014:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204567.790146:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204567.790172:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204567.790176:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204567.790202:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204567.982004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204568.301970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204568.301981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204568.301985:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204568.301995:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3699 00000100:00080000:0.0:1788204568.302008:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204568.302015:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204568.302059:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559975936/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204568.302087:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204568.302089:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204568.302090:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204568.302094:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3699 00000100:00080000:0.0:1788204568.302095:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204568.302098:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204568.302106:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204568.302120:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559976064/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204568.302127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204568.302130:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1788204568.302130:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559975936:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204568.302132:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204568.302138:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3699 00000100:00080000:0.0:1788204568.302141:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204568.302143:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204568.302146:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204568.302153:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559976192/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204568.302256:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559976064:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204568.302284:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559976192:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204569.005989:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204570.029986:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204571.054047:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788204571.374176:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788204571.374227:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788204571.374249:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204571.374307:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204571.374318:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:1.0:1788204571.374367:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204568/real 1788204571] req@ffff93c0e9ce5880 x1875064559976064/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204571.374400:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559976064/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204571.374414:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204571.374420:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204571.374424:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559976064:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204571.374434:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204568/real 1788204571] req@ffff93c0e9ce4e00 x1875064559976192/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204571.374447:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559976192/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204571.374458:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204571.374462:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204571.374478:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559976192:192.168.201.156@tcp:8:kworker.0 00000100:00100000:1.0:1788204571.374487:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204568/real 1788204571] req@ffff93c0e9ce6300 x1875064559975936/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204571.374499:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559975936/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204571.374513:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204571.374529:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204571.374533:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559975936:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788204572.077962:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204572.910035:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204572.910049:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204572.910143:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204572.910171:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204572.910174:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204572.910213:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204573.102021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204573.421934:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204573.421943:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204573.421946:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204573.421955:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3704 00000100:00080000:0.0:1788204573.421964:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204573.421969:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204573.421998:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559976320/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204573.422022:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204573.422024:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204573.422025:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204573.422029:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3704 00000100:00080000:0.0:1788204573.422031:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204573.422032:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204573.422039:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204573.422048:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559976448/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204573.422055:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204573.422058:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204573.422059:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204573.422062:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3704 00000100:00080000:0.0:1788204573.422064:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204573.422066:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204573.422070:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204573.422077:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559976576/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204573.422082:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559976320:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788204573.422147:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204573.422186:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204573/real 1788204573] req@ffff93c0e9ce7800 x1875064559976320/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204573.422204:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559976320/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204573.422215:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204573.422219:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204573.422222:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559976320:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204573.422229:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559976448:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204573.422262:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204573.422276:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204573/real 1788204573] req@ffff93c0e9ce5500 x1875064559976448/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204573.422296:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559976448/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204573.422304:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204573.422307:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204573.422310:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559976448:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204573.422341:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559976576:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204573.422364:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204573.422376:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204573/real 1788204573] req@ffff93c0e9ce5180 x1875064559976576/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204573.422388:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559976576/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204573.422397:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204573.422400:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204573.422403:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559976576:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204574.125976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204575.150046:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204576.174039:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204577.198017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204578.030006:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204578.030023:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204578.030143:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204578.030178:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204578.030182:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204578.030202:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204578.222056:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204578.541983:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204578.541997:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204578.542001:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204578.542015:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3709 00000100:00080000:0.0:1788204578.542028:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204578.542037:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204578.542085:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559976704/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204578.542123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204578.542128:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204578.542129:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204578.542134:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3709 00000100:00080000:0.0:1788204578.542137:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204578.542141:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204578.542154:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204578.542178:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559976832/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204578.542188:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204578.542192:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204578.542194:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204578.542199:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3709 00000100:00080000:0.0:1788204578.542203:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204578.542208:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204578.542215:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204578.542224:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559976704:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204578.542232:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559976960/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204578.542308:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204578.542351:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204578/real 1788204578] req@ffff93c0e9ce4a80 x1875064559976704/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204578.542369:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559976704/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204578.542379:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204578.542383:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204578.542387:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559976704:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204578.542395:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559976832:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204578.542417:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204578.542429:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204578/real 1788204578] req@ffff93c0e9ce7100 x1875064559976832/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204578.542437:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559976832/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204578.542442:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204578.542445:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204578.542447:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559976832:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204578.542468:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559976960:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204578.542482:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204578.542488:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204578/real 1788204578] req@ffff93c0e9ce5f80 x1875064559976960/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204578.542499:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559976960/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204578.542505:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204578.542506:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204578.542508:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559976960:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204579.246030:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204580.270069:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204581.294047:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204582.318000:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204583.150057:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204583.150089:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204583.150221:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204583.150248:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204583.150253:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204583.150279:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204583.342038:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204583.662003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204583.662016:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204583.662022:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204583.662034:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3714 00000100:00080000:0.0:1788204583.662046:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204583.662053:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204583.662117:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559977088/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204583.662151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204583.662154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204583.662156:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204583.662160:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3714 00000100:00080000:0.0:1788204583.662164:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204583.662168:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204583.662180:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204583.662193:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559977216/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204583.662205:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204583.662206:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559977088:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204583.662212:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204583.662216:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204583.662222:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3714 00000100:00080000:0.0:1788204583.662225:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204583.662229:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204583.662234:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204583.662246:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559977344/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204583.662294:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204583.662343:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204583/real 1788204583] req@ffff93c0e9ce7b80 x1875064559977088/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204583.662361:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559977088/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204583.662372:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204583.662376:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204583.662380:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559977088:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204583.662410:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559977216:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204583.662435:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204583.662449:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204583/real 1788204583] req@ffff93c0e9ce6d80 x1875064559977216/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204583.662462:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559977216/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204583.662472:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204583.662474:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204583.662476:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559977216:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204583.662482:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559977344:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204583.662502:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204583.662513:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204583/real 1788204583] req@ffff93c0e9ce5c00 x1875064559977344/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204583.662525:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559977344/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204583.662535:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204583.662539:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204583.662542:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559977344:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204584.365997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204585.389959:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204586.413972:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204587.438012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204588.270089:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204588.270104:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204588.270191:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204588.270217:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204588.270222:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204588.270248:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204588.462012:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204588.782069:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204588.782083:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204588.782089:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204588.782101:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3719 00000100:00080000:0.0:1788204588.782113:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204588.782122:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204588.782172:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559977472/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204588.782211:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204588.782214:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204588.782216:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204588.782221:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3719 00000100:00080000:0.0:1788204588.782224:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204588.782228:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204588.782239:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204588.782251:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559977600/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204588.782261:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204588.782265:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204588.782266:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204588.782271:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3719 00000100:00080000:0.0:1788204588.782274:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204588.782277:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204588.782282:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204588.782292:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559977728/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204588.782315:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559977472:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788204588.782389:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204588.782427:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204588/real 1788204588] req@ffff93c0e9ce4000 x1875064559977472/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204588.782440:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559977472/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204588.782447:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204588.782450:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204588.782453:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559977472:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204588.782459:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559977600:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204588.782472:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204588.782479:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204588/real 1788204588] req@ffff93c0e9ce6a00 x1875064559977600/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204588.782486:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559977600/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204588.782491:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204588.782492:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204588.782494:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559977600:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204588.782497:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559977728:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204588.782507:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204588.782511:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204588/real 1788204588] req@ffff93c0e9ce6680 x1875064559977728/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204588.782518:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559977728/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204588.782525:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204588.782527:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204588.782528:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559977728:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204589.486034:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204590.509988:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204591.533981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204592.558005:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204593.390030:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204593.390046:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204593.390134:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204593.390154:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204593.390157:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204593.390173:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204593.582010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204593.901966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204593.901974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204593.901977:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204593.901985:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3724 00000100:00080000:3.0:1788204593.901995:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204593.902001:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204593.902032:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f100 x1875064559977856/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204593.902053:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204593.902056:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204593.902057:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204593.902060:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3724 00000100:00080000:3.0:1788204593.902061:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204593.902064:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204593.902071:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204593.902078:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4df80 x1875064559977984/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204593.902084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204593.902086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204593.902088:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204593.902090:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3724 00000100:00080000:3.0:1788204593.902093:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204593.902095:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204593.902100:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204593.902110:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559977856:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788204593.902113:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064559978112/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204593.902168:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204593.902199:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204593/real 1788204593] req@ffff93c1c4a4f100 x1875064559977856/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204593.902211:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f100 x1875064559977856/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204593.902219:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204593.902221:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204593.902224:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559977856:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204593.902229:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559977984:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204593.902241:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204593.902248:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204593/real 1788204593] req@ffff93c1c4a4df80 x1875064559977984/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204593.902254:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4df80 x1875064559977984/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204593.902259:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204593.902261:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204593.902263:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559977984:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204593.902279:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559978112:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204593.902289:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204593.902294:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204593/real 1788204593] req@ffff93c1c4a4e680 x1875064559978112/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204593.902301:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064559978112/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204593.902311:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204593.902313:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204593.902314:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559978112:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204594.605961:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204595.630174:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204596.653971:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204597.678012:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204598.510012:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204598.510026:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204598.510103:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204598.510122:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204598.510125:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204598.510139:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204598.701934:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204599.021889:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204599.021897:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204599.021900:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204599.021907:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3729 00000100:00080000:3.0:1788204599.021915:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204599.021921:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204599.021954:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064559978240/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204599.021982:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204599.021985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204599.021987:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204599.021991:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3729 00000100:00080000:3.0:1788204599.021995:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204599.021999:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204599.022009:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204599.022022:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064559978368/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204599.022030:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204599.022032:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559978240:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788204599.022034:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204599.022037:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204599.022042:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3729 00000100:00080000:3.0:1788204599.022045:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204599.022048:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204599.022051:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788204599.022061:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064559978496/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204599.022093:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204599.022123:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204599/real 1788204599] req@ffff93c1c4a4dc00 x1875064559978240/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204599.022140:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064559978240/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204599.022149:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204599.022153:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204599.022157:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559978240:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204599.022179:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559978368:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204599.022195:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204599.022203:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204599/real 1788204599] req@ffff93c1c4a4ed80 x1875064559978368/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204599.022216:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064559978368/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204599.022221:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204599.022223:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204599.022225:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559978368:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204599.022228:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559978496:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204599.022238:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204599.022243:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204599/real 1788204599] req@ffff93c1c4a4fb80 x1875064559978496/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204599.022250:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064559978496/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204599.022255:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204599.022257:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204599.022259:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559978496:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204599.725981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204600.749934:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204601.774028:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204602.797964:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204603.629977:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204603.629993:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204603.630078:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204603.630103:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204603.630107:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204603.630132:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204603.821992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204604.141985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204604.141995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204604.141998:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204604.142008:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3735 00000100:00080000:1.0:1788204604.142018:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204604.142025:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204604.142059:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559978624/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204604.142081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204604.142084:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204604.142086:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204604.142089:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3735 00000100:00080000:1.0:1788204604.142092:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204604.142096:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204604.142104:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204604.142116:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559978752/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204604.142124:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204604.142127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204604.142128:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204604.142132:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3735 00000100:00080000:1.0:1788204604.142135:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204604.142136:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559978624:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204604.142138:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204604.142142:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204604.142151:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559978880/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204604.142197:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204604.142231:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204604/real 1788204604] req@ffff93c1c5037100 x1875064559978624/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204604.142243:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559978624/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204604.142252:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204604.142256:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204604.142259:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559978624:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204604.142288:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559978752:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204604.142309:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204604.142321:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204604/real 1788204604] req@ffff93c1c5034e00 x1875064559978752/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204604.142341:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559978752/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204604.142349:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204604.142352:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204604.142356:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559978752:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204604.142362:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559978880:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204604.142381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204604.142391:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204604/real 1788204604] req@ffff93c1c5034a80 x1875064559978880/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204604.142399:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559978880/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204604.142405:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204604.142407:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204604.142410:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559978880:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204604.845984:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204605.870028:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204606.893953:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204607.918018:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204608.750036:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204608.750051:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204608.750180:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204608.750206:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204608.750211:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204608.750249:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204608.942041:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204609.261960:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204609.261973:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204609.261977:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204609.261989:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3740 00000100:00080000:2.0:1788204609.262002:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204609.262011:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204609.262056:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559979008/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204609.262085:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204609.262089:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204609.262091:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204609.262095:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3740 00000100:00080000:2.0:1788204609.262100:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204609.262104:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204609.262115:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204609.262130:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559979136/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204609.262135:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559979008:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204609.262148:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204609.262152:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204609.262154:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204609.262159:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3740 00000100:00080000:2.0:1788204609.262163:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204609.262167:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204609.262173:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204609.262188:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559979264/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204609.262203:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204609.262242:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204609/real 1788204609] req@ffff93c0e62bb800 x1875064559979008/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204609.262255:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559979008/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204609.262263:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204609.262266:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204609.262270:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559979008:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204609.262299:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559979136:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204609.262322:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204609.262334:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204609/real 1788204609] req@ffff93c0e62b9500 x1875064559979136/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204609.262346:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559979136/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204609.262352:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204609.262354:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204609.262356:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559979136:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204609.262360:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559979264:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204609.262373:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204609.262380:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204609/real 1788204609] req@ffff93c0e62b9f80 x1875064559979264/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204609.262389:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559979264/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204609.262399:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204609.262402:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204609.262405:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559979264:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204609.966008:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204610.990008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204612.014027:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204613.038016:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204613.870011:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204613.870032:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204613.870141:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204613.870177:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204613.870180:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204613.870204:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204614.062024:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204614.382011:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204614.382030:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204614.382036:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204614.382048:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3745 00000100:00080000:0.0:1788204614.382061:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204614.382069:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204614.382115:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559979392/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204614.382144:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204614.382148:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204614.382150:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204614.382155:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3745 00000100:00080000:0.0:1788204614.382159:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204614.382163:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204614.382174:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204614.382187:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559979520/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204614.382192:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559979392:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204614.382205:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204614.382208:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204614.382210:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204614.382216:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3745 00000100:00080000:0.0:1788204614.382219:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204614.382223:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204614.382228:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204614.382238:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559979648/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204614.382276:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204614.382329:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204614/real 1788204614] req@ffff93c0e9ce4380 x1875064559979392/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204614.382344:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559979392/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204614.382355:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204614.382360:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204614.382364:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559979392:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204614.382404:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559979520:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204614.382424:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204614.382433:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204614/real 1788204614] req@ffff93c0e9ce4700 x1875064559979520/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204614.382441:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559979520/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204614.382446:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204614.382449:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204614.382451:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559979520:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204614.382455:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559979648:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204614.382465:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204614.382471:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204614/real 1788204614] req@ffff93c0e9ce7480 x1875064559979648/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204614.382478:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559979648/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204614.382485:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204614.382486:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204614.382488:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559979648:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204615.086054:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204616.110066:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204617.134050:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204618.157951:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204618.989993:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204618.990009:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204618.990116:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204618.990134:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204618.990137:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204618.990152:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204619.181994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204619.501959:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204619.501973:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204619.501978:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204619.501992:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3750 00000100:00080000:2.0:1788204619.502005:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204619.502013:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204619.502059:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559979776/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204619.502082:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204619.502084:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204619.502086:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204619.502089:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3750 00000100:00080000:2.0:1788204619.502092:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204619.502095:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204619.502104:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204619.502115:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559979904/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204619.502126:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559979776:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204619.502130:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204619.502136:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204619.502137:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204619.502141:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3750 00000100:00080000:2.0:1788204619.502143:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204619.502145:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204619.502148:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204619.502157:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559980032/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204619.502216:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204619.502258:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204619/real 1788204619] req@ffff93c0e62b8a80 x1875064559979776/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204619.502277:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559979776/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204619.502288:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204619.502292:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204619.502296:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559979776:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204619.502320:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559979904:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204619.502342:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204619.502354:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204619/real 1788204619] req@ffff93c0e62b9c00 x1875064559979904/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204619.502367:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559979904/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204619.502381:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204619.502384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204619.502387:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559979904:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204619.502394:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559980032:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204619.502412:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204619.502422:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204619/real 1788204619] req@ffff93c0e62bad80 x1875064559980032/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204619.502434:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559980032/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204619.502457:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204619.502462:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204619.502465:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559980032:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204620.206033:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204621.229995:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204622.254058:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204623.277971:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204624.109997:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204624.110013:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204624.110113:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204624.110135:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204624.110137:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204624.110152:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204624.302025:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204624.621938:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204624.621950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204624.621954:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204624.621963:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3755 00000100:00080000:3.0:1788204624.621983:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204624.621990:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204624.622030:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064559980160/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204624.622052:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204624.622055:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204624.622056:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204624.622059:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3755 00000100:00080000:3.0:1788204624.622062:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204624.622064:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204624.622075:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204624.622083:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064559980288/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204624.622091:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204624.622092:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559980160:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788204624.622096:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204624.622100:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204624.622104:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3755 00000100:00080000:3.0:1788204624.622105:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204624.622108:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204624.622111:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788204624.622120:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064559980416/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204624.622162:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204624.622199:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204624/real 1788204624] req@ffff93c1c4a4c000 x1875064559980160/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204624.622212:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064559980160/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204624.622220:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204624.622223:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204624.622226:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559980160:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204624.622247:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559980288:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204624.622261:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204624.622269:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204624/real 1788204624] req@ffff93c1c4a4ea00 x1875064559980288/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204624.622276:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064559980288/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204624.622281:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204624.622284:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204624.622286:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559980288:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204624.622289:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559980416:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204624.622299:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204624.622304:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204624/real 1788204624] req@ffff93c1c4a4f480 x1875064559980416/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204624.622310:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064559980416/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204624.622323:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204624.622325:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204624.622326:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559980416:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204625.326034:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204626.350054:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204627.373989:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204628.397950:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204629.230029:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204629.230046:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204629.230174:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204629.230202:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204629.230205:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204629.230224:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204629.422028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204629.741917:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204629.741930:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204629.741934:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204629.741947:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3760 00000100:00080000:1.0:1788204629.741959:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204629.741966:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204629.742008:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559980544/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204629.742035:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204629.742038:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204629.742039:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204629.742042:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3760 00000100:00080000:1.0:1788204629.742044:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204629.742047:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204629.742057:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204629.742065:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559980672/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204629.742071:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204629.742074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204629.742075:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204629.742079:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559980544:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204629.742081:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3760 00000100:00080000:1.0:1788204629.742087:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204629.742090:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204629.742093:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204629.742100:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559980800/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204629.742154:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204629.742192:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204629/real 1788204629] req@ffff93c1c5034a80 x1875064559980544/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204629.742204:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559980544/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204629.742216:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204629.742219:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204629.742222:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559980544:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204629.742247:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559980672:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204629.742261:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204629.742283:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204629/real 1788204629] req@ffff93c1c5034e00 x1875064559980672/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204629.742291:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559980672/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204629.742297:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204629.742298:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204629.742300:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559980672:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204629.742304:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559980800:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204629.742318:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204629.742324:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204629/real 1788204629] req@ffff93c1c5037100 x1875064559980800/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204629.742330:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559980800/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204629.742337:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204629.742339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204629.742340:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559980800:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204630.446055:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204631.469951:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204632.494111:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204633.518022:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204634.350026:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204634.350042:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204634.350150:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204634.350177:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204634.350181:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204634.350223:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204634.542028:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204634.862011:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204634.862026:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204634.862032:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204634.862044:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3765 00000100:00080000:3.0:1788204634.862057:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204634.862066:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204634.862110:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064559980928/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204634.862141:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204634.862146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204634.862148:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204634.862152:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3765 00000100:00080000:3.0:1788204634.862156:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204634.862160:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204634.862172:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204634.862200:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064559981056/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204634.862208:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559980928:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788204634.862215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204634.862219:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204634.862221:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204634.862227:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3765 00000100:00080000:3.0:1788204634.862231:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204634.862234:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204634.862240:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788204634.862252:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064559981184/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204634.862332:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559981056:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204634.862367:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559981184:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204635.566004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204636.590019:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204637.613975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788204637.934171:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788204637.934228:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788204637.934249:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204637.934300:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204637.934310:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788204637.934359:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204634/real 1788204637] req@ffff93c1c4a4c700 x1875064559981056/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204637.934388:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064559981056/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204637.934403:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204637.934409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204637.934414:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559981056:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204637.934423:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204634/real 1788204637] req@ffff93c1c4a4e300 x1875064559981184/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204637.934436:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064559981184/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204637.934447:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204637.934451:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204637.934454:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559981184:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788204637.934465:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204634/real 1788204637] req@ffff93c1c4a4c380 x1875064559980928/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204637.934478:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064559980928/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204637.934485:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204637.934489:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204637.934493:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559980928:192.168.201.156@tcp:250:kworker.0 00010000:00010000:2.0:1788204638.637986:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204639.470011:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204639.470023:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204639.470129:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204639.470156:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204639.470161:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204639.470190:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204639.661964:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204639.981942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204639.981950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204639.981952:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204639.981960:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3770 00000100:00080000:3.0:1788204639.981968:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204639.981973:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204639.981998:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064559981312/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204639.982020:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204639.982022:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204639.982023:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204639.982026:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3770 00000100:00080000:3.0:1788204639.982028:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204639.982031:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204639.982039:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204639.982049:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064559981440/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204639.982059:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204639.982062:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204639.982064:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204639.982068:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3770 00000100:00080000:3.0:1788204639.982072:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204639.982075:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204639.982080:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204639.982084:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559981312:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788204639.982106:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559981568/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204639.982160:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204639.982221:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204639/real 1788204639] req@ffff93c1c4a4d180 x1875064559981312/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204639.982241:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064559981312/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204639.982251:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204639.982256:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204639.982260:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559981312:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204639.982267:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559981440:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204639.982289:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204639.982301:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204639/real 1788204639] req@ffff93c1c4a4f800 x1875064559981440/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204639.982311:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064559981440/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204639.982319:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204639.982323:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204639.982326:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559981440:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204639.982352:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559981568:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204639.982370:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204639.982381:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204639/real 1788204639] req@ffff93c1c4a4d880 x1875064559981568/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204639.982391:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559981568/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204639.982401:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204639.982404:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204639.982407:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559981568:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204640.686009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204641.710009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204642.734008:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204643.758081:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204644.589972:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204644.589992:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204644.590081:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204644.590100:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204644.590103:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204644.590119:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204644.781999:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204645.101938:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204645.101945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204645.101947:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204645.101955:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3775 00000100:00080000:3.0:1788204645.101969:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204645.101978:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204645.102026:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ca80 x1875064559981696/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204645.102049:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204645.102052:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204645.102053:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204645.102056:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3775 00000100:00080000:3.0:1788204645.102058:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204645.102061:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204645.102068:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204645.102077:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d500 x1875064559981824/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204645.102083:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204645.102086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204645.102087:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204645.102089:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3775 00000100:00080000:3.0:1788204645.102091:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204645.102094:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204645.102097:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204645.102106:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559981696:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788204645.102107:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d880 x1875064559981952/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204645.102173:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204645.102207:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204645/real 1788204645] req@ffff93c1c4a4ca80 x1875064559981696/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204645.102223:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ca80 x1875064559981696/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204645.102234:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204645.102238:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204645.102242:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559981696:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204645.102248:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559981824:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204645.102277:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204645.102287:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204645/real 1788204645] req@ffff93c1c4a4d500 x1875064559981824/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204645.102295:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d500 x1875064559981824/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204645.102300:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204645.102302:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204645.102305:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559981824:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204645.102321:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559981952:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204645.102333:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204645.102339:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204645/real 1788204645] req@ffff93c1c4a4d880 x1875064559981952/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204645.102346:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d880 x1875064559981952/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204645.102352:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204645.102354:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204645.102356:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559981952:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204645.805954:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204646.830014:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204647.854013:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204648.878092:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204649.710060:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204649.710080:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204649.710204:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204649.710235:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204649.710239:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204649.710265:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204649.902048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204650.221966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204650.221980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204650.221985:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204650.221997:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3781 00000100:00080000:1.0:1788204650.222009:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204650.222018:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204650.222065:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559982080/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204650.222095:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204650.222099:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204650.222100:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204650.222104:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3781 00000100:00080000:1.0:1788204650.222107:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204650.222111:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204650.222123:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204650.222136:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559982208/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204650.222144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204650.222147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204650.222149:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204650.222151:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3781 00000100:00080000:1.0:1788204650.222154:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204650.222158:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204650.222163:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204650.222166:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559982080:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204650.222173:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559982336/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204650.222254:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204650.222310:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204650/real 1788204650] req@ffff93c1c5035c00 x1875064559982080/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204650.222327:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559982080/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204650.222335:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204650.222338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204650.222343:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559982080:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204650.222352:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559982208:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204650.222376:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204650.222389:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204650/real 1788204650] req@ffff93c1c5037800 x1875064559982208/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204650.222401:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559982208/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204650.222409:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204650.222412:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204650.222415:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559982208:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204650.222440:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559982336:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204650.222457:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204650.222473:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204650/real 1788204650] req@ffff93c1c5036680 x1875064559982336/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204650.222481:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559982336/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204650.222489:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204650.222491:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204650.222493:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559982336:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204650.926032:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204651.949958:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204652.974021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204653.998056:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204654.830029:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204654.830060:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204654.830154:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204654.830172:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204654.830175:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204654.830195:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204655.021959:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204655.341975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204655.341986:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204655.341991:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204655.342001:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3786 00000100:00080000:2.0:1788204655.342013:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204655.342022:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204655.342063:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559982464/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204655.342092:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204655.342097:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204655.342099:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204655.342103:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3786 00000100:00080000:2.0:1788204655.342107:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204655.342111:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204655.342121:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204655.342135:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559982592/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204655.342148:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788204655.342154:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559982464:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204655.342165:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204655.342168:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204655.342174:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3786 00000100:00080000:2.0:1788204655.342177:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204655.342181:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204655.342186:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204655.342199:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559982720/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204655.342237:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204655.342292:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204655/real 1788204655] req@ffff93c0e62bbb80 x1875064559982464/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204655.342313:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559982464/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204655.342325:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204655.342330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204655.342335:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559982464:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204655.342357:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559982592:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204655.342381:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204655.342393:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204655/real 1788204655] req@ffff93c0e62bb100 x1875064559982592/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204655.342415:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559982592/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204655.342424:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204655.342428:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204655.342431:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559982592:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204655.342438:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559982720:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204655.342457:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204655.342468:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204655/real 1788204655] req@ffff93c0e62b8700 x1875064559982720/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204655.342480:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559982720/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204655.342491:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204655.342494:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204655.342498:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559982720:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204656.046055:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204657.069966:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204658.094039:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204659.118082:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204659.950097:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204659.950118:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204659.950267:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204659.950311:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204659.950316:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204659.950364:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204660.141951:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204660.462026:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204660.462040:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204660.462046:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204660.462059:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3791 00000100:00080000:0.0:1788204660.462073:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204660.462082:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204660.462133:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559982848/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204660.462171:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204660.462175:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204660.462177:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204660.462182:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3791 00000100:00080000:0.0:1788204660.462185:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204660.462191:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204660.462203:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204660.462217:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559982976/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204660.462230:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204660.462238:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1788204660.462240:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559982848:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204660.462242:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204660.462259:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3791 00000100:00080000:0.0:1788204660.462264:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204660.462271:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204660.462278:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204660.462302:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559983104/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204660.462320:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204660.462380:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204660/real 1788204660] req@ffff93c0e9ce7480 x1875064559982848/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204660.462401:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559982848/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204660.462413:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204660.462416:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204660.462419:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559982848:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204660.462442:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559982976:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204660.462467:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204660.462481:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204660/real 1788204660] req@ffff93c0e9ce4700 x1875064559982976/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204660.462495:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559982976/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204660.462505:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204660.462509:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204660.462511:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559982976:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204660.462517:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559983104:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204660.462534:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204660.462542:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204660/real 1788204660] req@ffff93c0e9ce4380 x1875064559983104/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204660.462550:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559983104/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204660.462557:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204660.462559:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204660.462562:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559983104:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204661.166075:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204662.189975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204663.214075:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204664.237954:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204665.070110:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204665.070127:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204665.070237:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204665.070276:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204665.070282:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204665.070314:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204665.262021:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204665.581942:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204665.581950:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204665.581954:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204665.581963:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3796 00000100:00080000:0.0:1788204665.581973:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204665.581986:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204665.582022:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559983232/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204665.582048:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204665.582057:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204665.582059:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204665.582064:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3796 00000100:00080000:0.0:1788204665.582069:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204665.582073:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204665.582085:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204665.582100:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559983360/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204665.582107:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559983232:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204665.582113:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204665.582118:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204665.582119:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204665.582123:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3796 00000100:00080000:0.0:1788204665.582126:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204665.582129:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204665.582134:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204665.582143:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559983488/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204665.582180:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204665.582225:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204665/real 1788204665] req@ffff93c0e9ce6680 x1875064559983232/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204665.582247:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559983232/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204665.582260:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204665.582265:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204665.582268:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559983232:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204665.582297:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559983360:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204665.582316:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204665.582326:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204665/real 1788204665] req@ffff93c0e9ce6a00 x1875064559983360/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204665.582335:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559983360/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204665.582341:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204665.582344:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204665.582346:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559983360:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204665.582351:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559983488:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204665.582362:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204665.582368:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204665/real 1788204665] req@ffff93c0e9ce4000 x1875064559983488/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204665.582376:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559983488/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204665.582383:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204665.582385:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204665.582387:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559983488:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204666.285956:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204667.310003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204668.333998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204669.358079:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204670.190003:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204670.190022:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204670.190152:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204670.190176:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204670.190180:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204670.190209:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204670.381971:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204670.701965:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204670.701975:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204670.701980:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204670.701990:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3801 00000100:00080000:0.0:1788204670.702009:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204670.702015:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204670.702057:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559983616/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204670.702086:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204670.702088:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204670.702089:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204670.702092:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3801 00000100:00080000:0.0:1788204670.702094:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204670.702097:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204670.702106:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204670.702117:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559983744/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204670.702127:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204670.702130:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204670.702132:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204670.702137:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3801 00000100:00080000:0.0:1788204670.702140:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204670.702144:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204670.702150:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204670.702167:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559983872/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204670.702168:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559983616:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788204670.702246:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204670.702289:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204670/real 1788204670] req@ffff93c0e9ce5c00 x1875064559983616/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204670.702316:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559983616/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204670.702327:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204670.702331:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204670.702335:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559983616:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204670.702344:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559983744:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204670.702364:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204670.702373:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204670/real 1788204670] req@ffff93c0e9ce6d80 x1875064559983744/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204670.702380:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559983744/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204670.702386:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204670.702388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204670.702390:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559983744:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204670.702410:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559983872:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204670.702421:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204670.702426:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204670/real 1788204670] req@ffff93c0e9ce7b80 x1875064559983872/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204670.702433:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559983872/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204670.702443:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204670.702445:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204670.702446:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559983872:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204671.406018:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204672.429945:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204673.454064:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204674.478044:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204675.310021:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204675.310036:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204675.310151:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204675.310177:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204675.310181:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204675.310216:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204675.502017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204675.821970:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204675.821980:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204675.821984:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204675.821993:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3806 00000100:00080000:2.0:1788204675.822004:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204675.822011:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204675.822046:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559984000/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204675.822077:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204675.822080:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204675.822082:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204675.822086:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3806 00000100:00080000:2.0:1788204675.822089:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204675.822093:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204675.822102:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204675.822115:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559984128/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204675.822125:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204675.822129:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1788204675.822133:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559984000:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204675.822133:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204675.822142:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3806 00000100:00080000:2.0:1788204675.822146:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204675.822149:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204675.822154:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204675.822166:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559984256/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204675.822204:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204675.822245:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204675/real 1788204675] req@ffff93c0e62b8380 x1875064559984000/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204675.822260:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559984000/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204675.822267:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204675.822270:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204675.822273:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559984000:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204675.822297:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559984128:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204675.822312:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204675.822327:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204675/real 1788204675] req@ffff93c0e62b8000 x1875064559984128/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204675.822338:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559984128/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204675.822345:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204675.822349:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204675.822352:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559984128:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204675.822358:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559984256:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204675.822376:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204675.822383:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204675/real 1788204675] req@ffff93c0e62baa00 x1875064559984256/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204675.822390:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559984256/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204675.822396:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204675.822398:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204675.822400:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559984256:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204676.525958:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204677.550030:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204678.573989:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204679.598013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204680.430024:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204680.430038:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204680.430133:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204680.430159:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204680.430164:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204680.430190:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204680.621975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204680.941937:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204680.941950:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204680.941955:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204680.941967:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3811 00000100:00080000:0.0:1788204680.941980:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204680.941988:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204680.942033:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559984384/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204680.942058:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204680.942060:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204680.942061:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204680.942064:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3811 00000100:00080000:0.0:1788204680.942067:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204680.942070:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204680.942080:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204680.942088:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559984512/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204680.942096:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204680.942099:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204680.942100:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204680.942103:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3811 00000100:00080000:0.0:1788204680.942105:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204680.942108:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204680.942113:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204680.942113:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559984384:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204680.942121:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559984640/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204680.942181:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204680.942225:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204680/real 1788204680] req@ffff93c0e9ce5f80 x1875064559984384/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204680.942244:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559984384/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204680.942257:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204680.942260:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204680.942263:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559984384:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204680.942269:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559984512:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204680.942286:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204680.942295:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204680/real 1788204680] req@ffff93c0e9ce7100 x1875064559984512/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204680.942304:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559984512/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204680.942309:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204680.942311:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204680.942313:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559984512:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204680.942335:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559984640:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204680.942346:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204680.942352:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204680/real 1788204680] req@ffff93c0e9ce4a80 x1875064559984640/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204680.942359:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559984640/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204680.942369:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204680.942372:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204680.942375:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559984640:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204681.646005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204682.670022:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204683.693997:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204684.717986:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204685.550000:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204685.550013:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204685.550111:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204685.550137:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204685.550142:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204685.550169:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204685.742075:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204686.061962:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204686.061970:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204686.061974:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204686.061984:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3816 00000100:00080000:2.0:1788204686.061995:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204686.062002:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204686.062050:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559984768/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204686.062080:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204686.062085:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204686.062087:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204686.062091:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3816 00000100:00080000:2.0:1788204686.062095:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204686.062100:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204686.062109:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204686.062123:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559984896/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204686.062134:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204686.062137:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559984768:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204686.062139:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204686.062144:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204686.062150:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3816 00000100:00080000:2.0:1788204686.062154:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204686.062158:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204686.062164:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204686.062176:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559985024/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204686.062203:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204686.062236:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204686/real 1788204686] req@ffff93c0e62ba680 x1875064559984768/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204686.062249:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559984768/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204686.062260:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204686.062262:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204686.062272:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559984768:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204686.062290:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559984896:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204686.062339:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204686.062351:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204686/real 1788204686] req@ffff93c0e62bb480 x1875064559984896/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204686.062363:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559984896/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204686.062371:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204686.062373:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204686.062375:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559984896:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204686.062379:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559985024:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204686.062394:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204686.062400:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204686/real 1788204686] req@ffff93c0e62baa00 x1875064559985024/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204686.062407:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559985024/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204686.062413:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204686.062415:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204686.062417:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559985024:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204686.765974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204687.790007:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204688.813944:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204689.838013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204690.669989:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204690.670006:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204690.670109:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204690.670135:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204690.670140:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204690.670164:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204690.861974:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204691.181937:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204691.181944:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204691.181947:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204691.181954:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3822 00000100:00080000:0.0:1788204691.181964:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204691.181969:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204691.181997:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559985152/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204691.182019:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204691.182021:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204691.182022:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204691.182025:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3822 00000100:00080000:0.0:1788204691.182027:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204691.182030:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204691.182042:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204691.182054:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559985280/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204691.182061:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204691.182064:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204691.182065:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204691.182068:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3822 00000100:00080000:0.0:1788204691.182070:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204691.182073:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204691.182077:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559985152:192.168.201.156@tcp:250:kworker.0 00000008:00100000:0.0:1788204691.182079:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204691.182090:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559985408/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204691.182143:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204691.182181:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204691/real 1788204691] req@ffff93c0e9ce5180 x1875064559985152/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204691.182197:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559985152/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204691.182235:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204691.182239:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204691.182243:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559985152:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204691.182250:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559985280:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204691.182269:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204691.182280:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204691/real 1788204691] req@ffff93c0e9ce5500 x1875064559985280/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204691.182290:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559985280/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204691.182298:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204691.182301:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204691.182304:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559985280:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204691.182324:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559985408:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204691.182340:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204691.182348:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204691/real 1788204691] req@ffff93c0e9ce7800 x1875064559985408/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204691.182359:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559985408/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204691.182367:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204691.182370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204691.182373:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559985408:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204691.886005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204692.910006:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204693.934014:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204694.957972:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204695.789996:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204695.790011:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204695.790099:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204695.790162:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204695.790168:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204695.790198:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204695.981954:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204696.301954:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204696.301961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204696.301965:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204696.301972:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3827 00000100:00080000:1.0:1788204696.301981:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204696.301987:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204696.302025:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559985536/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204696.302058:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204696.302062:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204696.302064:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204696.302069:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3827 00000100:00080000:1.0:1788204696.302073:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204696.302076:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204696.302085:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204696.302097:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559985664/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204696.302106:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204696.302109:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204696.302111:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204696.302114:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3827 00000100:00080000:1.0:1788204696.302129:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204696.302133:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204696.302138:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204696.302147:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559985792/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204696.302182:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559985536:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788204696.302249:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204696.302282:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204696/real 1788204696] req@ffff93c1c5036680 x1875064559985536/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204696.302295:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559985536/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204696.302302:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204696.302304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204696.302307:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559985536:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204696.302311:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559985664:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204696.302325:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204696.302333:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204696/real 1788204696] req@ffff93c1c5037800 x1875064559985664/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204696.302340:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559985664/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204696.302345:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204696.302347:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204696.302350:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559985664:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204696.302353:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559985792:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204696.302363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204696.302370:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204696/real 1788204696] req@ffff93c1c5035c00 x1875064559985792/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204696.302381:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559985792/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204696.302390:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204696.302394:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204696.302398:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559985792:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204697.005984:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204698.030064:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204699.054019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204700.078030:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204700.910025:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204700.910053:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204700.910186:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204700.910211:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204700.910215:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204700.910240:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204701.102029:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204701.421935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204701.421948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204701.421952:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204701.421964:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3832 00000100:00080000:1.0:1788204701.421977:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204701.421985:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204701.422053:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559985920/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204701.422079:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204701.422083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204701.422084:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204701.422087:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3832 00000100:00080000:1.0:1788204701.422090:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204701.422093:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204701.422102:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204701.422112:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559986048/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204701.422119:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204701.422122:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204701.422125:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204701.422127:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559985920:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204701.422129:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3832 00000100:00080000:1.0:1788204701.422134:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204701.422138:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204701.422141:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204701.422148:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559986176/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204701.422265:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559986048:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204701.422297:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559986176:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204702.126051:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204703.149954:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204704.174036:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788204704.494123:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788204704.494171:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788204704.494188:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204704.494227:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204704.494235:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788204704.494328:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204701/real 1788204704] req@ffff93c1c5034e00 x1875064559986048/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204704.494353:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559986048/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204704.494370:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204704.494376:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204704.494380:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559986048:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204704.494388:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204701/real 1788204704] req@ffff93c1c5034a80 x1875064559986176/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204704.494399:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559986176/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204704.494409:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204704.494412:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204704.494415:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559986176:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788204704.494421:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204701/real 1788204704] req@ffff93c1c5037100 x1875064559985920/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204704.494431:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559985920/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204704.494437:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204704.494440:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204704.494443:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559985920:192.168.201.156@tcp:250:kworker.0 00010000:00010000:0.0:1788204705.198019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204706.030053:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204706.030068:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204706.030150:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204706.030177:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204706.030180:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204706.030197:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204706.222017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204706.541981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204706.541994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204706.541999:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204706.542012:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3837 00000100:00080000:1.0:1788204706.542025:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204706.542033:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204706.542071:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559986304/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204706.542105:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204706.542109:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204706.542111:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204706.542116:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3837 00000100:00080000:1.0:1788204706.542119:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204706.542123:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204706.542134:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204706.542146:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559986432/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204706.542157:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204706.542161:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204706.542163:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204706.542167:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3837 00000100:00080000:1.0:1788204706.542170:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204706.542173:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204706.542177:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204706.542185:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559986560/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204706.542243:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559986304:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788204706.542354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204706.542407:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204706/real 1788204706] req@ffff93c1c5037100 x1875064559986304/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204706.542425:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559986304/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204706.542436:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204706.542439:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204706.542443:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559986304:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204706.542451:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559986432:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204706.542473:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204706.542485:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204706/real 1788204706] req@ffff93c1c5034a80 x1875064559986432/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204706.542495:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559986432/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204706.542501:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204706.542504:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204706.542507:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559986432:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204706.542512:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559986560:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204706.542530:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204706.542540:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204706/real 1788204706] req@ffff93c1c5034e00 x1875064559986560/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204706.542549:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559986560/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204706.542565:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204706.542568:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204706.542571:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559986560:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204707.246000:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204708.270025:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204709.293990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204710.318020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204711.150019:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204711.150031:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204711.150122:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204711.150139:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204711.150142:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204711.150160:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204711.341959:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204711.661974:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204711.661983:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204711.661987:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204711.661996:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3842 00000100:00080000:2.0:1788204711.662008:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204711.662016:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204711.662056:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559986688/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204711.662086:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204711.662090:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204711.662092:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204711.662096:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3842 00000100:00080000:2.0:1788204711.662100:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204711.662104:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204711.662112:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204711.662122:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559986816/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204711.662131:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559986688:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204711.662131:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204711.662138:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204711.662141:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204711.662146:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3842 00000100:00080000:2.0:1788204711.662149:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204711.662152:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204711.662158:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204711.662171:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559986944/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204711.662199:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204711.662238:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204711/real 1788204711] req@ffff93c0e62b8000 x1875064559986688/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204711.662254:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559986688/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204711.662264:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204711.662268:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204711.662271:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559986688:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204711.662294:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559986816:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204711.662314:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204711.662327:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204711/real 1788204711] req@ffff93c0e62b8380 x1875064559986816/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204711.662339:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559986816/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204711.662351:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204711.662353:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204711.662356:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559986816:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204711.662360:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559986944:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204711.662374:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204711.662380:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204711/real 1788204711] req@ffff93c0e62b8700 x1875064559986944/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204711.662387:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559986944/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204711.662393:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204711.662396:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204711.662398:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559986944:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204712.366031:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204713.390007:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204714.414018:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204715.437966:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204716.270073:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204716.270099:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204716.270213:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204716.270242:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204716.270247:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204716.270280:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204716.462053:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204716.781979:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204716.781993:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204716.781999:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204716.782011:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3847 00000100:00080000:0.0:1788204716.782024:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204716.782032:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204716.782080:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559987072/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204716.782119:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204716.782122:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204716.782124:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204716.782128:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3847 00000100:00080000:0.0:1788204716.782131:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204716.782134:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204716.782144:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204716.782156:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559987200/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204716.782165:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204716.782169:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1788204716.782169:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559987072:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204716.782172:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204716.782180:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3847 00000100:00080000:0.0:1788204716.782183:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204716.782186:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204716.782189:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204716.782199:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559987328/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204716.782263:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204716.782325:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204716/real 1788204716] req@ffff93c0e9ce6300 x1875064559987072/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204716.782349:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559987072/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204716.782358:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204716.782363:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204716.782369:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559987072:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204716.782399:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559987200:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204716.782427:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204716.782441:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204716/real 1788204716] req@ffff93c0e9ce4e00 x1875064559987200/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204716.782455:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559987200/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204716.782464:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204716.782467:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204716.782471:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559987200:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204716.782476:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559987328:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204716.782491:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204716.782500:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204716/real 1788204716] req@ffff93c0e9ce5880 x1875064559987328/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204716.782508:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559987328/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204716.782515:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204716.782517:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204716.782519:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559987328:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204717.486082:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204718.510009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204719.534000:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204720.558116:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204721.390029:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204721.390057:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204721.390170:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204721.390191:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204721.390194:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204721.390210:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204721.581986:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204721.901934:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204721.901945:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204721.901948:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204721.901959:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3852 00000100:00080000:0.0:1788204721.901971:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204721.901980:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204721.902025:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064559987456/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204721.902049:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204721.902052:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204721.902053:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204721.902057:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3852 00000100:00080000:0.0:1788204721.902060:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204721.902062:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204721.902072:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204721.902082:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064559987584/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204721.902090:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204721.902093:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1788204721.902093:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559987456:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204721.902095:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204721.902103:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3852 00000100:00080000:0.0:1788204721.902105:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204721.902107:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204721.902111:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204721.902119:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064559987712/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204721.902161:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204721.902206:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204721/real 1788204721] req@ffff93c0e9ce5880 x1875064559987456/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204721.902221:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064559987456/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204721.902229:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204721.902232:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204721.902235:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559987456:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204721.902263:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559987584:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204721.902286:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204721.902299:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204721/real 1788204721] req@ffff93c0e9ce4e00 x1875064559987584/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204721.902311:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064559987584/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204721.902321:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204721.902323:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204721.902325:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559987584:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204721.902329:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559987712:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204721.902344:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204721.902350:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204721/real 1788204721] req@ffff93c0e9ce6300 x1875064559987712/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204721.902357:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064559987712/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204721.902364:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204721.902366:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204721.902368:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559987712:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204722.606009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204723.630009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204724.654014:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204725.678092:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204726.510015:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204726.510027:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204726.510108:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204726.510132:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204726.510136:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204726.510163:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204726.701991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204727.021928:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204727.021937:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204727.021941:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204727.021954:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3857 00000100:00080000:2.0:1788204727.021968:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204727.021975:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204727.022009:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559987840/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204727.022041:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204727.022047:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204727.022051:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204727.022056:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3857 00000100:00080000:2.0:1788204727.022059:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204727.022063:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204727.022073:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204727.022085:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559987968/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204727.022094:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559987840:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204727.022096:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204727.022103:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204727.022105:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204727.022109:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3857 00000100:00080000:2.0:1788204727.022112:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204727.022115:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204727.022120:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204727.022129:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559988096/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204727.022164:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204727.022204:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204727/real 1788204727] req@ffff93c0e62bb100 x1875064559987840/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204727.022217:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559987840/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204727.022225:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204727.022229:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204727.022232:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559987840:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204727.022250:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559987968:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204727.022264:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204727.022271:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204727/real 1788204727] req@ffff93c0e62bbb80 x1875064559987968/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204727.022278:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559987968/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204727.022283:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204727.022285:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204727.022287:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559987968:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204727.022290:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559988096:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204727.022299:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204727.022304:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204727/real 1788204727] req@ffff93c0e62bad80 x1875064559988096/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204727.022310:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559988096/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204727.022316:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204727.022317:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204727.022319:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559988096:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204727.725977:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204728.749982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204729.773965:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204730.798007:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204731.630022:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204731.630035:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204731.630139:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204731.630171:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204731.630176:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204731.630201:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204731.821989:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204732.141954:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204732.141962:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204732.141965:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204732.141973:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3863 00000100:00080000:1.0:1788204732.141981:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204732.141986:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204732.142014:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559988224/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204732.142038:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204732.142041:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204732.142042:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204732.142046:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3863 00000100:00080000:1.0:1788204732.142049:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204732.142052:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204732.142061:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204732.142072:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559988352/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204732.142081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204732.142084:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204732.142085:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204732.142089:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3863 00000100:00100000:0.0:1788204732.142090:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559988224:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204732.142092:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204732.142096:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204732.142099:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204732.142108:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559988480/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204732.142151:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204732.142195:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204732/real 1788204732] req@ffff93c1c5035c00 x1875064559988224/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204732.142213:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559988224/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204732.142223:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204732.142227:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204732.142231:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559988224:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204732.142238:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559988352:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204732.142255:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204732.142264:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204732/real 1788204732] req@ffff93c1c5037800 x1875064559988352/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204732.142271:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559988352/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204732.142276:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204732.142278:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204732.142280:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559988352:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204732.142296:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559988480:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204732.142306:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204732.142311:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204732/real 1788204732] req@ffff93c1c5036680 x1875064559988480/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204732.142318:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559988480/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204732.142324:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204732.142333:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204732.142335:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559988480:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204732.845991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204733.869967:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204734.893987:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204735.917960:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204736.749952:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204736.749964:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204736.750067:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204736.750090:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204736.750094:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204736.750114:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204736.942049:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204737.261923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204737.261935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204737.261939:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204737.261949:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3868 00000100:00080000:2.0:1788204737.261958:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204737.261965:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204737.262004:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559988608/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204737.262034:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204737.262038:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204737.262040:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204737.262045:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3868 00000100:00080000:2.0:1788204737.262049:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204737.262054:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204737.262065:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204737.262083:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559988736/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204737.262087:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559988608:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204737.262102:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204737.262105:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204737.262107:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204737.262111:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3868 00000100:00080000:2.0:1788204737.262114:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204737.262117:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204737.262122:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204737.262134:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559988864/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204737.262156:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204737.262208:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204737/real 1788204737] req@ffff93c0e62b9c00 x1875064559988608/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204737.262231:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559988608/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204737.262240:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204737.262243:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204737.262246:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559988608:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204737.262266:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559988736:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204737.262285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204737.262294:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204737/real 1788204737] req@ffff93c0e62b8a80 x1875064559988736/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204737.262302:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559988736/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204737.262308:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204737.262310:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204737.262312:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559988736:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204737.262316:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559988864:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204737.262327:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204737.262333:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204737/real 1788204737] req@ffff93c0e62b9f80 x1875064559988864/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204737.262345:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559988864/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204737.262351:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204737.262353:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204737.262355:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559988864:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204737.965954:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204738.990029:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204740.013939:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204741.037995:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204741.870034:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204741.870050:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204741.870140:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204741.870158:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204741.870161:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204741.870180:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204742.061948:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204742.381954:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204742.381966:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204742.381970:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204742.381978:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3873 00000100:00080000:2.0:1788204742.381989:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204742.381995:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204742.382027:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559988992/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204742.382057:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204742.382060:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204742.382062:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204742.382065:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3873 00000100:00080000:2.0:1788204742.382068:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204742.382070:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204742.382078:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204742.382087:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559989120/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204742.382094:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204742.382097:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204742.382099:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204742.382102:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3873 00000100:00080000:2.0:1788204742.382104:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204742.382107:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204742.382110:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204742.382119:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559989248/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204742.382133:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559988992:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788204742.382219:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204742.382279:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204742/real 1788204742] req@ffff93c0e62b9500 x1875064559988992/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204742.382300:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559988992/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204742.382313:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204742.382318:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204742.382340:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559988992:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204742.382348:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559989120:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204742.382374:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204742.382388:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204742/real 1788204742] req@ffff93c0e62bb800 x1875064559989120/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204742.382409:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559989120/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204742.382418:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204742.382421:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204742.382425:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559989120:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204742.382453:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559989248:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204742.382475:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204742.382488:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204742/real 1788204742] req@ffff93c0e62b9f80 x1875064559989248/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204742.382500:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559989248/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204742.382510:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204742.382514:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204742.382517:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559989248:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204743.086079:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204744.109995:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204745.134077:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204746.158002:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204746.990034:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204746.990052:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204746.990164:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204746.990183:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204746.990186:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204746.990203:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204747.182034:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204747.501951:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204747.501962:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204747.501966:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204747.501977:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3878 00000100:00080000:2.0:1788204747.501990:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204747.501998:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204747.502048:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559989376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204747.502083:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204747.502088:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204747.502090:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204747.502095:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3878 00000100:00080000:2.0:1788204747.502099:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204747.502104:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204747.502115:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204747.502141:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559989376:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788204747.502145:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559989504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204747.502167:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204747.502172:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204747.502174:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204747.502180:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3878 00000100:00080000:2.0:1788204747.502184:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204747.502189:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204747.502195:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204747.502209:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559989632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204747.502224:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204747.502297:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204747/real 1788204747] req@ffff93c0e62b8a80 x1875064559989376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204747.502313:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559989376/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204747.502323:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204747.502326:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204747.502329:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559989376:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204747.502362:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559989504:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204747.502380:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204747.502390:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204747/real 1788204747] req@ffff93c0e62b9c00 x1875064559989504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204747.502398:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559989504/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204747.502404:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204747.502407:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204747.502410:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559989504:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204747.502417:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559989632:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204747.502437:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204747.502449:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204747/real 1788204747] req@ffff93c0e62bad80 x1875064559989632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204747.502462:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559989632/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204747.502480:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204747.502483:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204747.502485:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559989632:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204748.206007:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204749.230106:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204750.253950:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204751.278063:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204752.110050:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204752.110071:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204752.110252:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204752.110299:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204752.110305:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204752.110337:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204752.301960:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204752.621967:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204752.621979:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204752.621984:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204752.621996:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3883 00000100:00080000:2.0:1788204752.622010:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204752.622019:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204752.622071:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559989760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204752.622105:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204752.622108:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204752.622110:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204752.622115:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3883 00000100:00080000:2.0:1788204752.622118:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204752.622122:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204752.622134:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204752.622160:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559989888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204752.622172:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559989760:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204752.622174:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204752.622182:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204752.622185:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204752.622190:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3883 00000100:00080000:2.0:1788204752.622194:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204752.622198:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204752.622204:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204752.622217:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559990016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204752.622258:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204752.622319:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204752/real 1788204752] req@ffff93c0e62bbb80 x1875064559989760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204752.622341:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559989760/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204752.622353:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204752.622357:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204752.622362:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559989760:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204752.622388:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559989888:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204752.622409:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204752.622423:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204752/real 1788204752] req@ffff93c0e62bb100 x1875064559989888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204752.622435:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559989888/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204752.622443:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204752.622446:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204752.622449:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559989888:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204752.622455:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559990016:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204752.622471:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204752.622480:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204752/real 1788204752] req@ffff93c0e62b8700 x1875064559990016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204752.622490:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559990016/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204752.622500:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204752.622503:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204752.622506:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559990016:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204753.326058:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204754.349985:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204755.374049:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204756.398004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204757.230062:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204757.230091:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204757.230217:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204757.230238:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204757.230241:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204757.230257:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204757.422038:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204757.741978:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204757.741988:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204757.741992:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204757.742002:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3888 00000100:00080000:0.0:1788204757.742013:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204757.742020:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204757.742097:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559990144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204757.742124:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204757.742128:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204757.742129:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204757.742133:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3888 00000100:00080000:0.0:1788204757.742136:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204757.742140:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204757.742152:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204757.742164:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559990272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204757.742173:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204757.742176:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204757.742178:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204757.742178:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559990144:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204757.742181:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3888 00000100:00080000:0.0:1788204757.742184:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204757.742187:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204757.742191:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204757.742200:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559990400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204757.742259:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204757.742316:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204757/real 1788204757] req@ffff93c0e9ce7800 x1875064559990144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204757.742331:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559990144/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204757.742339:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204757.742342:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204757.742345:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559990144:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204757.742368:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559990272:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204757.742383:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204757.742391:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204757/real 1788204757] req@ffff93c0e9ce5500 x1875064559990272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204757.742399:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559990272/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204757.742405:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204757.742407:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204757.742408:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559990272:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204757.742411:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559990400:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204757.742422:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204757.742426:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204757/real 1788204757] req@ffff93c0e9ce5180 x1875064559990400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204757.742433:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559990400/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204757.742440:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204757.742442:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204757.742444:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559990400:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204758.445950:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204759.470017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204760.494003:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204761.518075:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204762.350064:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204762.350078:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204762.350166:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204762.350184:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204762.350187:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204762.350204:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204762.541964:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204762.862018:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204762.862030:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204762.862035:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204762.862046:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3893 00000100:00080000:2.0:1788204762.862058:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204762.862065:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204762.862118:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559990528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204762.862154:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204762.862157:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204762.862158:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204762.862162:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3893 00000100:00080000:2.0:1788204762.862165:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204762.862168:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204762.862179:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204762.862193:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559990656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204762.862201:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204762.862204:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204762.862206:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204762.862210:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3893 00000100:00080000:2.0:1788204762.862212:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204762.862214:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204762.862217:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204762.862227:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559990528:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788204762.862227:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559990784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204762.862322:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204762.862379:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204762/real 1788204762] req@ffff93c0e62b8380 x1875064559990528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204762.862401:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559990528/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204762.862415:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204762.862418:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204762.862421:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559990528:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204762.862427:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559990656:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204762.862447:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204762.862457:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204762/real 1788204762] req@ffff93c0e62b8000 x1875064559990656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204762.862465:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559990656/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204762.862471:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204762.862473:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204762.862475:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559990656:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204762.862497:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559990784:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204762.862509:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204762.862516:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204762/real 1788204762] req@ffff93c0e62baa00 x1875064559990784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204762.862524:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559990784/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204762.862531:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204762.862533:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204762.862535:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559990784:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204763.566058:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204764.589964:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204765.614051:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204766.638011:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204767.470026:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204767.470044:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204767.470180:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204767.470205:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204767.470208:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204767.470239:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204767.662085:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204767.982034:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204767.982045:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204767.982049:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204767.982062:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3898 00000100:00080000:2.0:1788204767.982074:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204767.982083:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204767.982128:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559990912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204767.982154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204767.982157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204767.982158:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204767.982161:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3898 00000100:00080000:2.0:1788204767.982169:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204767.982172:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204767.982181:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204767.982191:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559991040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204767.982200:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204767.982203:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204767.982204:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204767.982209:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3898 00000100:00080000:2.0:1788204767.982213:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788204767.982214:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559990912:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204767.982218:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204767.982226:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204767.982238:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559991168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204767.982343:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559991040:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204767.982370:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559991168:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204768.686064:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204769.710028:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204770.734009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204771.054163:0:2331:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:2.0:1788204771.054211:0:2331:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788204771.054235:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:2.0:1788204771.054280:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:2.0:1788204771.054291:0:2331:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:1.0:1788204771.054343:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204767/real 1788204771] req@ffff93c0e62ba680 x1875064559991040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204771.054369:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559991040/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204771.054378:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204771.054382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204771.054386:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559991040:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204771.054392:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204767/real 1788204771] req@ffff93c0e62baa00 x1875064559991168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204771.054400:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559991168/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204771.054407:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204771.054409:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204771.054411:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559991168:192.168.201.156@tcp:8:kworker.0 00000100:00100000:1.0:1788204771.054416:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204767/real 1788204771] req@ffff93c0e62bb480 x1875064559990912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204771.054425:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559990912/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204771.054430:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204771.054433:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204771.054435:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559990912:192.168.201.156@tcp:250:kworker.0 00010000:00010000:3.0:1788204771.757955:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204772.589999:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204772.590017:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204772.590113:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204772.590137:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204772.590141:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204772.590171:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204772.781996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204773.102037:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204773.102051:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204773.102056:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204773.102066:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3903 00000100:00080000:2.0:1788204773.102078:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204773.102084:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204773.102120:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559991296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204773.102153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204773.102158:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204773.102161:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204773.102166:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3903 00000100:00080000:2.0:1788204773.102170:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204773.102175:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204773.102185:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204773.102200:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559991424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204773.102213:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204773.102217:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559991296:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204773.102218:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204773.102223:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204773.102231:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3903 00000100:00080000:2.0:1788204773.102234:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204773.102238:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204773.102243:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204773.102264:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559991552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204773.102299:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204773.102348:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204773/real 1788204773] req@ffff93c0e62b8000 x1875064559991296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204773.102370:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559991296/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204773.102383:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204773.102388:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204773.102393:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559991296:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204773.102421:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559991424:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204773.102446:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204773.102459:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204773/real 1788204773] req@ffff93c0e62b8380 x1875064559991424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204773.102473:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559991424/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204773.102481:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204773.102484:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204773.102487:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559991424:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204773.102494:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559991552:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204773.102516:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204773.102528:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204773/real 1788204773] req@ffff93c0e62b8700 x1875064559991552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204773.102539:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559991552/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204773.102549:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204773.102553:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204773.102563:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559991552:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204773.806080:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204774.830021:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204775.854033:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204776.877967:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204777.709989:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204777.710004:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204777.710097:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204777.710124:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204777.710127:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204777.710151:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204777.902000:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204778.221953:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204778.221961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204778.221970:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204778.221978:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3909 00000100:00080000:1.0:1788204778.221992:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204778.221999:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204778.222038:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559991680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204778.222065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204778.222068:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204778.222069:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204778.222072:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3909 00000100:00080000:1.0:1788204778.222074:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204778.222077:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204778.222084:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204778.222094:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559991808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204778.222100:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204778.222103:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204778.222104:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204778.222106:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3909 00000100:00080000:1.0:1788204778.222108:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204778.222111:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204778.222115:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559991680:192.168.201.156@tcp:250:kworker.0 00000008:00100000:1.0:1788204778.222116:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204778.222125:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559991936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204778.222178:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204778.222214:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204778/real 1788204778] req@ffff93c1c5036680 x1875064559991680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204778.222231:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559991680/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204778.222241:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204778.222244:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204778.222248:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559991680:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204778.222255:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559991808:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204778.222272:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204778.222285:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204778/real 1788204778] req@ffff93c1c5037800 x1875064559991808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204778.222296:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559991808/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204778.222304:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204778.222308:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204778.222311:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559991808:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204778.222331:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559991936:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204778.222348:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204778.222357:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204778/real 1788204778] req@ffff93c1c5035c00 x1875064559991936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204778.222368:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559991936/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204778.222378:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204778.222381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204778.222384:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559991936:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204778.925922:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204779.950014:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204780.973976:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204781.998057:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204782.830008:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204782.830027:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204782.830135:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204782.830156:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204782.830158:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204782.830174:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204783.021907:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204783.341997:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204783.342011:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204783.342015:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204783.342035:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3914 00000100:00080000:0.0:1788204783.342047:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204783.342059:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204783.342104:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559992064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204783.342140:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204783.342143:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204783.342144:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204783.342148:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3914 00000100:00080000:0.0:1788204783.342152:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204783.342155:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204783.342166:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204783.342177:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559992192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204783.342188:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204783.342188:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559992064:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204783.342194:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204783.342198:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204783.342202:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3914 00000100:00080000:0.0:1788204783.342206:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204783.342208:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204783.342213:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204783.342230:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559992320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204783.342266:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204783.342312:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204783/real 1788204783] req@ffff93c0e9ce4a80 x1875064559992064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204783.342333:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559992064/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204783.342343:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204783.342348:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204783.342352:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559992064:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204783.342381:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559992192:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204783.342404:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204783.342414:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204783/real 1788204783] req@ffff93c0e9ce7100 x1875064559992192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204783.342422:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559992192/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204783.342427:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204783.342429:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204783.342431:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559992192:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204783.342435:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559992320:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204783.342447:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204783.342453:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204783/real 1788204783] req@ffff93c0e9ce5f80 x1875064559992320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204783.342460:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559992320/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204783.342471:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204783.342473:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204783.342475:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559992320:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204784.046031:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204785.070006:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204786.094107:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204787.117941:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204787.950030:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204787.950049:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204787.950159:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204787.950186:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204787.950191:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204787.950217:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204788.142105:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204788.461926:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204788.461936:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204788.461940:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204788.461950:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3919 00000100:00080000:2.0:1788204788.461960:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204788.461966:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204788.462006:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559992448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204788.462030:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204788.462033:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204788.462034:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204788.462037:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3919 00000100:00080000:2.0:1788204788.462039:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204788.462041:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204788.462051:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204788.462059:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559992576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204788.462066:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204788.462068:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204788.462070:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204788.462073:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3919 00000100:00080000:2.0:1788204788.462075:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204788.462078:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204788.462082:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204788.462083:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559992448:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788204788.462092:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559992704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204788.462148:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204788.462202:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204788/real 1788204788] req@ffff93c0e62bb100 x1875064559992448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204788.462222:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559992448/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204788.462233:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204788.462238:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204788.462241:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559992448:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204788.462249:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559992576:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204788.462270:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204788.462279:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204788/real 1788204788] req@ffff93c0e62bbb80 x1875064559992576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204788.462287:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559992576/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204788.462293:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204788.462295:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204788.462297:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559992576:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204788.462326:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559992704:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204788.462347:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204788.462358:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204788/real 1788204788] req@ffff93c0e62bad80 x1875064559992704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204788.462370:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559992704/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204788.462380:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204788.462384:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204788.462387:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559992704:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204789.165960:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204790.189983:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204791.213943:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204792.238017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204793.070005:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204793.070021:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204793.070141:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204793.070162:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204793.070165:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204793.070182:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204793.262024:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204793.581922:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204793.581932:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204793.581937:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204793.581947:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3924 00000100:00080000:2.0:1788204793.581958:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204793.581965:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204793.582003:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559992832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204793.582031:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204793.582035:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204793.582037:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204793.582041:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3924 00000100:00080000:2.0:1788204793.582046:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204793.582050:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204793.582059:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204793.582072:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559992960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204793.582083:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204793.582087:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204793.582089:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204793.582095:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559992832:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204793.582097:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3924 00000100:00080000:2.0:1788204793.582103:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204793.582109:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204793.582113:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204793.582126:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559993088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204793.582163:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204793.582208:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204793/real 1788204793] req@ffff93c0e62b9c00 x1875064559992832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204793.582227:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559992832/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204793.582238:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204793.582242:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204793.582245:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559992832:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204793.582252:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559992960:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204793.582270:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204793.582282:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204793/real 1788204793] req@ffff93c0e62b8a80 x1875064559992960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204793.582294:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559992960/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204793.582301:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204793.582304:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204793.582307:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559992960:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204793.582331:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559993088:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204793.582348:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204793.582358:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204793/real 1788204793] req@ffff93c0e62b9f80 x1875064559993088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204793.582369:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559993088/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204793.582378:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204793.582381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204793.582384:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559993088:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204794.285977:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204795.309975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204796.334061:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204797.358051:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204798.190012:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204798.190024:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204798.190116:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204798.190142:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204798.190147:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204798.190170:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204798.381996:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204798.701961:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204798.701970:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204798.701974:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204798.701992:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3929 00000100:00080000:0.0:1788204798.702005:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204798.702012:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204798.702050:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559993216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204798.702078:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204798.702081:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204798.702083:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204798.702086:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3929 00000100:00080000:0.0:1788204798.702089:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204798.702092:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204798.702101:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204798.702113:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559993344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204798.702123:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204798.702126:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204798.702128:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204798.702133:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3929 00000100:00080000:0.0:1788204798.702138:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788204798.702139:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559993216:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204798.702144:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204798.702151:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204798.702163:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559993472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204798.702209:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204798.702262:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204798/real 1788204798] req@ffff93c0e9ce7b80 x1875064559993216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204798.702282:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559993216/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204798.702298:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204798.702303:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204798.702307:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559993216:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204798.702314:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559993344:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204798.702336:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204798.702377:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204798/real 1788204798] req@ffff93c0e9ce6d80 x1875064559993344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204798.702390:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559993344/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204798.702398:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204798.702402:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204798.702405:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559993344:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204798.702430:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559993472:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204798.702451:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204798.702463:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204798/real 1788204798] req@ffff93c0e9ce5c00 x1875064559993472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204798.702476:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559993472/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204798.702493:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204798.702497:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204798.702500:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559993472:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204799.405975:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204800.430023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204801.453951:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204802.478031:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204803.310040:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204803.310052:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204803.310132:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204803.310160:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204803.310165:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204803.310186:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204803.502000:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204803.821971:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204803.821982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204803.821985:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204803.821996:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3934 00000100:00080000:2.0:1788204803.822007:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204803.822015:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204803.822051:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064559993600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204803.822081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204803.822084:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204803.822086:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204803.822090:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3934 00000100:00080000:2.0:1788204803.822094:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204803.822097:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204803.822107:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204803.822121:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064559993728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204803.822140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204803.822144:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204803.822145:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204803.822152:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3934 00000100:00100000:1.0:1788204803.822159:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559993600:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204803.822160:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204803.822172:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204803.822181:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204803.822197:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064559993856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204803.822252:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204803.822339:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204803/real 1788204803] req@ffff93c0e62bb800 x1875064559993600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204803.822367:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064559993600/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204803.822381:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204803.822386:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204803.822390:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559993600:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204803.822398:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559993728:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204803.822419:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204803.822430:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204803/real 1788204803] req@ffff93c0e62b9500 x1875064559993728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204803.822446:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064559993728/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204803.822454:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204803.822457:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204803.822460:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559993728:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204803.822486:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559993856:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204803.822504:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204803.822513:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204803/real 1788204803] req@ffff93c0e62b9f80 x1875064559993856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204803.822523:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064559993856/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204803.822533:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204803.822536:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204803.822538:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559993856:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204804.526067:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204805.550035:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204806.573998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204807.598023:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204808.430048:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204808.430068:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204808.430174:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204808.430212:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204808.430216:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204808.430245:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788204808.621981:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204808.941950:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204808.941961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204808.941965:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204808.941976:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3939 00000100:00080000:2.0:1788204808.941987:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204808.941994:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204808.942034:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064559993984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204808.942057:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204808.942060:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204808.942061:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204808.942064:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3939 00000100:00080000:2.0:1788204808.942065:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204808.942068:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204808.942077:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204808.942085:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064559994112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204808.942093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204808.942096:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204808.942097:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204808.942102:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559993984:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204808.942102:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3939 00000100:00080000:2.0:1788204808.942104:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204808.942110:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204808.942114:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204808.942119:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064559994240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204808.942181:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204808.942221:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204808/real 1788204808] req@ffff93c0e62b8a80 x1875064559993984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204808.942235:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064559993984/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204808.942242:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204808.942245:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204808.942248:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559993984:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204808.942268:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559994112:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204808.942284:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204808.942292:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204808/real 1788204808] req@ffff93c0e62b9c00 x1875064559994112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204808.942304:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064559994112/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204808.942311:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204808.942315:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204808.942318:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559994112:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204808.942324:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559994240:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204808.942350:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204808.942358:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204808/real 1788204808] req@ffff93c0e62bad80 x1875064559994240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204808.942365:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064559994240/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204808.942372:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204808.942374:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204808.942376:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559994240:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204809.646069:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204810.669972:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204811.693982:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204812.718068:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204813.550019:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204813.550035:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204813.550154:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204813.550177:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204813.550181:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204813.550200:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204813.742036:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204814.061978:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204814.061989:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204814.061993:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204814.062005:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3944 00000100:00080000:0.0:1788204814.062019:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204814.062027:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204814.062076:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559994368/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204814.062116:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204814.062121:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204814.062122:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204814.062126:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3944 00000100:00080000:0.0:1788204814.062129:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204814.062133:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204814.062142:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204814.062152:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559994496/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204814.062158:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559994368:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204814.062162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204814.062166:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204814.062168:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204814.062173:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3944 00000100:00080000:0.0:1788204814.062176:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204814.062180:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204814.062185:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204814.062200:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559994624/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204814.062225:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204814.062271:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204814/real 1788204814] req@ffff93c0e9ce4000 x1875064559994368/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204814.062289:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559994368/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204814.062297:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204814.062300:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204814.062302:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559994368:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204814.062324:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559994496:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204814.062338:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204814.062346:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204814/real 1788204814] req@ffff93c0e9ce6a00 x1875064559994496/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204814.062353:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559994496/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204814.062358:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204814.062360:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204814.062362:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559994496:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204814.062365:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559994624:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204814.062375:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204814.062380:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204814/real 1788204814] req@ffff93c0e9ce6680 x1875064559994624/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204814.062387:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559994624/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204814.062397:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204814.062399:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204814.062401:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559994624:192.168.201.156@tcp:8:kworker.0 00010000:00010000:3.0:1788204814.766024:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204815.789996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788204816.814020:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204817.837996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204818.670035:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204818.670050:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204818.670164:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204818.670190:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204818.670193:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204818.670220:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204818.862069:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204819.181960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204819.181970:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204819.181975:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204819.181984:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3950 00000100:00080000:0.0:1788204819.182010:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204819.182019:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204819.182061:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559994752/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204819.182088:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204819.182090:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204819.182092:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204819.182095:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3950 00000100:00080000:0.0:1788204819.182097:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204819.182099:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204819.182109:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204819.182123:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559994880/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204819.182131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204819.182135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204819.182137:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204819.182138:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559994752:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204819.182141:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3950 00000100:00080000:0.0:1788204819.182144:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204819.182150:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204819.182153:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204819.182161:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559995008/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204819.182217:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204819.182270:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204819/real 1788204819] req@ffff93c0e9ce4380 x1875064559994752/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204819.182290:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559994752/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204819.182299:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204819.182303:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204819.182307:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559994752:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204819.182337:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559994880:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204819.182369:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204819.182379:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204819/real 1788204819] req@ffff93c0e9ce4700 x1875064559994880/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204819.182387:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559994880/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204819.182393:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204819.182395:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204819.182397:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559994880:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204819.182402:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559995008:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204819.182416:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204819.182426:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204819/real 1788204819] req@ffff93c0e9ce7480 x1875064559995008/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204819.182444:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559995008/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204819.182454:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204819.182457:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204819.182461:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559995008:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204819.885962:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204820.910047:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204821.934023:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204822.958067:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204823.790029:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204823.790050:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204823.790195:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204823.790228:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204823.790233:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204823.790262:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204823.982041:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204824.302008:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204824.302022:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204824.302027:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204824.302038:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3955 00000100:00080000:2.0:1788204824.302051:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204824.302059:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204824.302110:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559995136/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204824.302132:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204824.302135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204824.302136:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204824.302139:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3955 00000100:00080000:2.0:1788204824.302141:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204824.302144:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204824.302152:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204824.302161:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559995264/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204824.302167:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204824.302170:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204824.302171:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204824.302174:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3955 00000100:00080000:2.0:1788204824.302176:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204824.302178:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204824.302181:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204824.302186:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559995392/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204824.302192:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559995136:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788204824.302270:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204824.302318:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204824/real 1788204824] req@ffff93c0e62bbb80 x1875064559995136/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204824.302335:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559995136/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204824.302346:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204824.302350:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204824.302354:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559995136:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204824.302361:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559995264:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204824.302382:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204824.302393:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204824/real 1788204824] req@ffff93c0e62bb100 x1875064559995264/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204824.302404:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559995264/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204824.302411:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204824.302414:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204824.302417:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559995264:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204824.302442:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559995392:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204824.302471:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204824.302482:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204824/real 1788204824] req@ffff93c0e62b8700 x1875064559995392/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204824.302494:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559995392/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204824.302505:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204824.302508:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204824.302511:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559995392:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204825.006044:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204826.030017:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204827.054015:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204828.078004:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204828.910032:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204828.910048:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204828.910152:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204828.910179:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204828.910183:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204828.910211:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204829.102037:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204829.421984:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204829.421993:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204829.421997:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204829.422007:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3960 00000100:00080000:0.0:1788204829.422018:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204829.422025:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204829.422064:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7480 x1875064559995520/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204829.422091:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204829.422094:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204829.422095:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204829.422098:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3960 00000100:00080000:0.0:1788204829.422101:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204829.422104:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204829.422110:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204829.422120:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4700 x1875064559995648/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204829.422127:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204829.422130:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204829.422132:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204829.422136:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3960 00000100:00080000:0.0:1788204829.422139:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204829.422143:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204829.422147:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204829.422159:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4380 x1875064559995776/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204829.422170:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559995520:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788204829.422245:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204829.422289:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204829/real 1788204829] req@ffff93c0e9ce7480 x1875064559995520/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204829.422308:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7480 x1875064559995520/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204829.422325:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204829.422330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204829.422334:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559995520:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204829.422342:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559995648:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204829.422360:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204829.422369:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204829/real 1788204829] req@ffff93c0e9ce4700 x1875064559995648/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204829.422377:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4700 x1875064559995648/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204829.422383:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204829.422385:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204829.422387:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559995648:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204829.422411:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559995776:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204829.422430:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204829.422440:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204829/real 1788204829] req@ffff93c0e9ce4380 x1875064559995776/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204829.422452:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4380 x1875064559995776/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204829.422463:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204829.422466:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204829.422468:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559995776:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204830.125952:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204831.150011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204832.174031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204833.197999:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204834.030021:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204834.030039:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204834.030169:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204834.030196:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204834.030200:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204834.030227:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204834.222028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204834.541986:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204834.541996:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204834.542001:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204834.542011:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3965 00000100:00080000:2.0:1788204834.542023:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204834.542030:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204834.542068:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559995904/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204834.542097:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204834.542101:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204834.542103:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204834.542107:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3965 00000100:00080000:2.0:1788204834.542111:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204834.542114:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204834.542124:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204834.542137:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559996032/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204834.542146:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559995904:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204834.542153:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204834.542157:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204834.542159:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204834.542165:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3965 00000100:00080000:2.0:1788204834.542169:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204834.542173:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204834.542178:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204834.542191:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559996160/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204834.542232:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559996032:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204834.542252:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559996160:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204835.245978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204836.270036:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204837.293933:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788204837.614171:0:2329:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788204837.614222:0:2329:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788204837.614245:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204837.614296:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204837.614309:0:2329:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:1.0:1788204837.614428:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204834/real 1788204837] req@ffff93c0e62b8000 x1875064559996032/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204837.614459:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559996032/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204837.614474:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204837.614480:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204837.614485:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559996032:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204837.614494:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204834/real 1788204837] req@ffff93c0e62bb480 x1875064559996160/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204837.614513:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559996160/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204837.614525:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204837.614529:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204837.614532:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559996160:192.168.201.156@tcp:8:kworker.0 00000100:00100000:1.0:1788204837.614541:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204834/real 1788204837] req@ffff93c0e62b8380 x1875064559995904/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204837.614553:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559995904/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204837.614561:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204837.614565:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204837.614568:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559995904:192.168.201.156@tcp:250:kworker.0 00010000:00010000:2.0:1788204838.318087:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204839.150005:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204839.150018:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204839.150124:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204839.150148:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204839.150152:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204839.150178:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204839.342016:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204839.661999:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204839.662014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204839.662019:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204839.662031:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3970 00000100:00080000:0.0:1788204839.662044:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204839.662052:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204839.662094:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6680 x1875064559996288/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204839.662133:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204839.662136:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204839.662138:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204839.662143:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3970 00000100:00080000:0.0:1788204839.662147:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204839.662151:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204839.662163:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204839.662184:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6a00 x1875064559996416/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204839.662201:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204839.662205:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559996288:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204839.662206:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204839.662209:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204839.662218:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3970 00000100:00080000:0.0:1788204839.662221:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204839.662225:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204839.662231:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204839.662244:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4000 x1875064559996544/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204839.662299:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204839.662354:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204839/real 1788204839] req@ffff93c0e9ce6680 x1875064559996288/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204839.662375:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6680 x1875064559996288/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204839.662388:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204839.662393:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204839.662399:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559996288:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204839.662433:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559996416:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204839.662456:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204839.662470:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204839/real 1788204839] req@ffff93c0e9ce6a00 x1875064559996416/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204839.662482:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6a00 x1875064559996416/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204839.662491:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204839.662495:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204839.662498:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559996416:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204839.662505:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559996544:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204839.662523:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204839.662533:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204839/real 1788204839] req@ffff93c0e9ce4000 x1875064559996544/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204839.662551:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4000 x1875064559996544/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204839.662565:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204839.662568:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204839.662572:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559996544:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204840.366026:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204841.390006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204842.413983:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204843.437968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204844.270051:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204844.270069:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204844.270164:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204844.270190:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204844.270194:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204844.270220:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204844.461992:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204844.782012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204844.782023:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204844.782028:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204844.782038:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3975 00000100:00080000:2.0:1788204844.782048:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204844.782055:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204844.782092:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064559996672/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204844.782118:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204844.782123:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204844.782125:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204844.782130:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3975 00000100:00080000:2.0:1788204844.782134:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204844.782138:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204844.782148:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559996672:192.168.201.156@tcp:250:kworker.0 00000008:00100000:2.0:1788204844.782148:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204844.782172:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064559996800/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204844.782183:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204844.782187:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204844.782189:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204844.782193:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3975 00000100:00080000:2.0:1788204844.782197:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204844.782201:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204844.782207:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000800:00000100:1.0:1788204844.782215:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:2.0:1788204844.782224:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064559996928/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204844.782257:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204844/real 1788204844] req@ffff93c0e62baa00 x1875064559996672/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204844.782277:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064559996672/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204844.782287:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204844.782291:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204844.782295:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559996672:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204844.782317:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559996800:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204844.782337:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204844.782347:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204844/real 1788204844] req@ffff93c0e62ba680 x1875064559996800/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204844.782359:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064559996800/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204844.782367:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204844.782369:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204844.782373:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559996800:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204844.782378:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559996928:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204844.782393:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204844.782400:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204844/real 1788204844] req@ffff93c0e62b8380 x1875064559996928/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204844.782410:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064559996928/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204844.782418:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204844.782421:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204844.782424:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559996928:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204845.485968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204846.510029:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204847.533946:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204848.558030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204849.390051:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204849.390064:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204849.390154:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204849.390180:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204849.390184:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204849.390211:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204849.581936:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204849.901964:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204849.901972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204849.901975:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204849.901982:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3980 00000100:00080000:0.0:1788204849.901991:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204849.901996:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204849.902026:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5c00 x1875064559997056/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204849.902047:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204849.902049:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204849.902050:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204849.902052:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3980 00000100:00080000:0.0:1788204849.902060:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204849.902063:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204849.902072:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204849.902085:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6d80 x1875064559997184/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204849.902101:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204849.902105:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559997056:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204849.902106:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204849.902110:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204849.902117:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3980 00000100:00080000:0.0:1788204849.902120:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204849.902122:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204849.902126:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204849.902136:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7b80 x1875064559997312/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204849.902173:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204849.902209:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204849/real 1788204849] req@ffff93c0e9ce5c00 x1875064559997056/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204849.902225:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5c00 x1875064559997056/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204849.902239:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204849.902243:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204849.902246:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559997056:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204849.902267:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559997184:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204849.902285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204849.902295:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204849/real 1788204849] req@ffff93c0e9ce6d80 x1875064559997184/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204849.902305:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6d80 x1875064559997184/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204849.902313:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204849.902316:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204849.902319:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559997184:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204849.902324:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559997312:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204849.902340:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204849.902349:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204849/real 1788204849] req@ffff93c0e9ce7b80 x1875064559997312/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204849.902359:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7b80 x1875064559997312/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204849.902368:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204849.902371:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204849.902373:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559997312:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204850.606055:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204851.629977:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204852.653993:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204853.678011:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204854.510016:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204854.510034:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204854.510143:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204854.510173:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204854.510177:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204854.510197:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204854.701964:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204855.021886:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204855.021899:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204855.021904:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204855.021926:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3985 00000100:00080000:0.0:1788204855.021940:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204855.021948:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204855.021992:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064559997440/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204855.022020:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204855.022023:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204855.022026:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204855.022030:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3985 00000100:00080000:0.0:1788204855.022034:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204855.022037:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204855.022049:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204855.022067:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064559997568/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204855.022071:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559997440:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204855.022087:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204855.022091:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204855.022093:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204855.022098:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3985 00000100:00080000:0.0:1788204855.022102:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204855.022107:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204855.022112:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204855.022124:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064559997696/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204855.022147:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204855.022192:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204855/real 1788204855] req@ffff93c0e9ce5f80 x1875064559997440/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204855.022206:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064559997440/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204855.022214:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204855.022218:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204855.022220:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559997440:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204855.022242:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559997568:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204855.022258:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204855.022266:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204855/real 1788204855] req@ffff93c0e9ce7100 x1875064559997568/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204855.022273:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064559997568/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204855.022279:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204855.022281:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204855.022283:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559997568:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204855.022287:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559997696:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204855.022297:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204855.022302:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204855/real 1788204855] req@ffff93c0e9ce4a80 x1875064559997696/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204855.022310:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064559997696/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204855.022317:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204855.022318:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204855.022320:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559997696:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204855.725999:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204856.750040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204857.773973:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204858.798009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204859.630017:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204859.630032:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204859.630134:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204859.630161:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204859.630166:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204859.630187:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204859.821936:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204860.141914:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204860.141925:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204860.141929:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204860.141939:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3991 00000100:00080000:0.0:1788204860.141950:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204860.141955:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204860.141986:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064559997824/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204860.142009:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204860.142012:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204860.142014:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204860.142018:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3991 00000100:00080000:0.0:1788204860.142021:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204860.142025:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204860.142033:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204860.142046:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064559997952/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204860.142055:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204860.142058:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204860.142060:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788204860.142066:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559997824:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204860.142067:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3991 00000100:00080000:0.0:1788204860.142074:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204860.142079:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204860.142085:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204860.142100:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064559998080/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204860.142127:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204860.142160:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204860/real 1788204860] req@ffff93c0e9ce5180 x1875064559997824/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204860.142171:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064559997824/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204860.142178:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204860.142180:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204860.142183:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559997824:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204860.142187:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559997952:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204860.142201:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204860.142207:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204860/real 1788204860] req@ffff93c0e9ce5500 x1875064559997952/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204860.142213:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064559997952/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204860.142222:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204860.142224:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204860.142225:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559997952:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204860.142242:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559998080:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204860.142251:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204860.142256:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204860/real 1788204860] req@ffff93c0e9ce7800 x1875064559998080/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204860.142269:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064559998080/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204860.142279:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204860.142282:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204860.142294:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559998080:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204860.845990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204861.869961:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204862.894040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204863.918045:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204864.750073:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204864.750092:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204864.750220:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204864.750248:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204864.750253:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204864.750279:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204864.941991:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204865.261975:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204865.261991:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204865.261997:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204865.262006:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 3996 00000100:00080000:1.0:1788204865.262017:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204865.262024:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204865.262064:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559998208/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204865.262086:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204865.262090:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204865.262091:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204865.262096:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3996 00000100:00080000:1.0:1788204865.262099:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204865.262103:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204865.262112:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204865.262120:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559998208:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204865.262124:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559998336/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204865.262133:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204865.262135:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204865.262136:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204865.262139:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 3996 00000100:00080000:1.0:1788204865.262142:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204865.262144:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204865.262146:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204865.262154:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559998464/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204865.262188:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204865.262267:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204865/real 1788204865] req@ffff93c1c5034e00 x1875064559998208/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204865.262286:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559998208/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204865.262296:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204865.262301:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204865.262305:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559998208:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204865.262340:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559998336:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204865.262363:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204865.262384:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204865/real 1788204865] req@ffff93c1c5034a80 x1875064559998336/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204865.262396:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559998336/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204865.262411:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204865.262414:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204865.262418:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559998336:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204865.262424:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559998464:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204865.262439:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204865.262447:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204865/real 1788204865] req@ffff93c1c5037100 x1875064559998464/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204865.262454:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559998464/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204865.262468:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204865.262470:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204865.262472:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559998464:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204865.966078:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204866.989997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204868.014008:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204869.038046:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204869.870022:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204869.870038:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204869.870150:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204869.870183:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204869.870187:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204869.870207:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204870.062018:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204870.381956:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204870.381967:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204870.381971:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204870.381981:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4001 00000100:00080000:1.0:1788204870.381993:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204870.382000:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204870.382046:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064559998592/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204870.382070:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204870.382073:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204870.382074:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204870.382077:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4001 00000100:00080000:1.0:1788204870.382079:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204870.382081:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204870.382090:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204870.382099:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064559998720/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204870.382106:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204870.382120:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204870.382122:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204870.382126:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4001 00000100:00080000:1.0:1788204870.382128:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204870.382131:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204870.382135:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204870.382141:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559998592:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204870.382146:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064559998848/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204870.382207:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204870.382249:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204870/real 1788204870] req@ffff93c1c5037100 x1875064559998592/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204870.382262:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064559998592/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204870.382274:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204870.382277:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204870.382281:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559998592:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204870.382285:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559998720:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204870.382298:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204870.382306:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204870/real 1788204870] req@ffff93c1c5034a80 x1875064559998720/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204870.382315:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064559998720/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204870.382323:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204870.382326:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204870.382329:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559998720:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204870.382357:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559998848:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204870.382373:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204870.382380:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204870/real 1788204870] req@ffff93c1c5034e00 x1875064559998848/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204870.382387:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064559998848/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204870.382395:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204870.382397:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204870.382399:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559998848:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204871.086051:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204872.109939:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204873.134031:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204874.157978:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204874.990025:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204874.990059:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204874.990168:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204874.990198:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204874.990203:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204874.990237:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204875.182008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204875.501985:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204875.501999:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204875.502004:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204875.502031:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4006 00000100:00080000:1.0:1788204875.502045:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204875.502053:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204875.502101:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064559998976/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204875.502133:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204875.502136:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204875.502137:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204875.502141:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4006 00000100:00080000:1.0:1788204875.502143:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204875.502145:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204875.502155:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204875.502165:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064559999104/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204875.502177:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204875.502179:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204875.502181:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204875.502186:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4006 00000100:00100000:0.0:1788204875.502189:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559998976:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204875.502189:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204875.502193:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204875.502199:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204875.502205:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064559999232/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204875.502264:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204875.502305:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204875/real 1788204875] req@ffff93c1c5035c00 x1875064559998976/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204875.502319:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064559998976/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204875.502327:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204875.502330:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204875.502333:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559998976:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204875.502355:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559999104:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204875.502368:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204875.502375:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204875/real 1788204875] req@ffff93c1c5037800 x1875064559999104/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204875.502384:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064559999104/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204875.502392:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204875.502395:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204875.502398:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559999104:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204875.502404:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559999232:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204875.502426:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204875.502436:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204875/real 1788204875] req@ffff93c1c5036680 x1875064559999232/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204875.502446:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064559999232/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204875.502456:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204875.502460:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204875.502463:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559999232:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204876.205913:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204877.230012:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204878.254023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204879.278092:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204880.110052:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204880.110072:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204880.110203:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204880.110232:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204880.110236:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204880.110264:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204880.301985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204880.622000:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204880.622017:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204880.622023:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204880.622035:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4011 00000100:00080000:2.0:1788204880.622050:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204880.622060:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204880.622102:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064559999360/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204880.622128:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204880.622131:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204880.622134:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204880.622139:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4011 00000100:00080000:2.0:1788204880.622143:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204880.622148:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204880.622160:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204880.622165:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559999360:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788204880.622176:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064559999488/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204880.622188:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204880.622191:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204880.622192:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204880.622197:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4011 00000100:00080000:2.0:1788204880.622200:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204880.622203:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204880.622208:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204880.622219:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064559999616/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204880.622236:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204880.622278:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204880/real 1788204880] req@ffff93c0e62bb480 x1875064559999360/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204880.622293:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064559999360/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204880.622304:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204880.622309:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204880.622313:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559999360:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204880.622340:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559999488:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204880.622364:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204880.622378:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204880/real 1788204880] req@ffff93c0e62b8000 x1875064559999488/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204880.622398:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064559999488/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204880.622404:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204880.622406:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204880.622409:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559999488:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204880.622413:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559999616:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204880.622428:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204880.622436:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204880/real 1788204880] req@ffff93c0e62b8700 x1875064559999616/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204880.622443:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064559999616/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204880.622450:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204880.622452:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204880.622454:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559999616:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204881.326035:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204882.350013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204883.374024:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204884.398002:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204885.230092:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204885.230109:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204885.230231:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204885.230256:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204885.230259:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204885.230275:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204885.422092:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204885.741961:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204885.741980:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204885.741983:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204885.741993:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4016 00000100:00080000:2.0:1788204885.742004:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204885.742010:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204885.742042:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064559999744/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204885.742066:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204885.742069:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204885.742070:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204885.742073:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4016 00000100:00080000:2.0:1788204885.742076:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204885.742078:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204885.742086:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204885.742094:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064559999872/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204885.742102:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204885.742105:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204885.742108:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204885.742111:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4016 00000100:00080000:2.0:1788204885.742114:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204885.742116:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204885.742121:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204885.742128:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064560000000/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204885.742152:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559999744:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788204885.742245:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204885.742300:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204885/real 1788204885] req@ffff93c0e62bb100 x1875064559999744/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204885.742320:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064559999744/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204885.742328:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204885.742332:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204885.742335:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064559999744:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204885.742341:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559999872:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204885.742356:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204885.742364:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204885/real 1788204885] req@ffff93c0e62bbb80 x1875064559999872/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204885.742371:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064559999872/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204885.742376:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204885.742378:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204885.742381:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064559999872:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204885.742404:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560000000:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204885.742416:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204885.742421:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204885/real 1788204885] req@ffff93c0e62bad80 x1875064560000000/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204885.742429:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064560000000/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204885.742439:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204885.742441:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204885.742443:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560000000:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204886.446006:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204887.470048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204888.494007:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204889.518060:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204890.350015:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204890.350031:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204890.350124:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204890.350151:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204890.350156:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204890.350187:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204890.542003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204890.861966:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204890.861982:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204890.861987:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204890.862001:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4021 00000100:00080000:1.0:1788204890.862015:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204890.862024:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204890.862077:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064560000128/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204890.862108:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204890.862112:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204890.862114:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204890.862119:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4021 00000100:00080000:1.0:1788204890.862123:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204890.862127:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204890.862138:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204890.862150:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064560000256/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204890.862161:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560000128:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204890.862163:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204890.862181:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204890.862183:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204890.862189:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4021 00000100:00080000:1.0:1788204890.862194:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204890.862198:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204890.862204:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204890.862218:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064560000384/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204890.862246:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204890.862296:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204890/real 1788204890] req@ffff93c1c5036680 x1875064560000128/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204890.862314:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064560000128/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204890.862335:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204890.862339:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204890.862343:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560000128:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204890.862369:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560000256:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204890.862396:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204890.862409:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204890/real 1788204890] req@ffff93c1c5037800 x1875064560000256/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204890.862421:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064560000256/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204890.862434:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204890.862437:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204890.862440:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560000256:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204890.862446:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560000384:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204890.862463:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204890.862471:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204890/real 1788204890] req@ffff93c1c5035c00 x1875064560000384/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204890.862481:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064560000384/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204890.862491:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204890.862500:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204890.862503:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560000384:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204891.565958:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204892.590003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204893.613990:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204894.637987:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204895.469963:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204895.469980:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204895.470096:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204895.470130:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204895.470135:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204895.470171:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204895.662052:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204895.981967:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204895.981977:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204895.981981:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204895.981992:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4026 00000100:00080000:1.0:1788204895.982004:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204895.982012:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204895.982054:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064560000512/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204895.982090:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204895.982093:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204895.982095:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204895.982098:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4026 00000100:00080000:1.0:1788204895.982103:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204895.982107:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204895.982117:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204895.982134:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064560000640/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204895.982149:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560000512:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204895.982155:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204895.982160:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204895.982161:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204895.982165:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4026 00000100:00080000:1.0:1788204895.982168:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204895.982171:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204895.982176:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204895.982186:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064560000768/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204895.982228:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204895.982272:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204895/real 1788204895] req@ffff93c1c5034e00 x1875064560000512/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204895.982299:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064560000512/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204895.982310:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204895.982314:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204895.982317:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560000512:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204895.982337:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560000640:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204895.982357:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204895.982367:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204895/real 1788204895] req@ffff93c1c5034a80 x1875064560000640/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204895.982375:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064560000640/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204895.982381:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204895.982383:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204895.982385:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560000640:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204895.982389:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560000768:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204895.982400:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204895.982406:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204895/real 1788204895] req@ffff93c1c5037100 x1875064560000768/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204895.982413:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064560000768/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204895.982420:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204895.982422:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204895.982424:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560000768:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204896.685987:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204897.710085:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204898.734037:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204899.758085:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204900.590065:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204900.590078:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204900.590174:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204900.590213:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204900.590220:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204900.590261:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204900.782056:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204901.102009:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204901.102019:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204901.102022:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204901.102031:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4031 00000100:00080000:2.0:1788204901.102040:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204901.102046:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204901.102079:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064560000896/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204901.102105:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204901.102109:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204901.102111:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204901.102115:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4031 00000100:00080000:2.0:1788204901.102118:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204901.102121:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204901.102129:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204901.102138:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064560001024/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204901.102146:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204901.102148:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204901.102150:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204901.102153:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4031 00000100:00080000:2.0:1788204901.102156:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204901.102158:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204901.102161:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204901.102169:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064560001152/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204901.102191:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560000896:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204901.102327:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560001024:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204901.102369:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560001152:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204901.806040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204902.829984:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204903.854071:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788204904.174132:0:2328:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788204904.174180:0:2328:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788204904.174199:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204904.174238:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204904.174246:0:2328:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:0.0:1788204904.174304:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204901/real 1788204904] req@ffff93c0e62b9f80 x1875064560001152/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204904.174333:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064560001152/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204904.174358:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204904.174365:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204904.174370:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560001152:192.168.201.156@tcp:8:kworker.0 00000100:00100000:0.0:1788204904.174379:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204901/real 1788204904] req@ffff93c0e62b9c00 x1875064560000896/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204904.174393:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064560000896/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204904.174416:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204904.174421:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204904.174424:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560000896:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204904.174433:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204901/real 1788204904] req@ffff93c0e62b8a80 x1875064560001024/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204904.174455:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064560001024/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204904.174464:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204904.174468:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204904.174472:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560001024:192.168.201.156@tcp:38:kworker.0 00010000:00010000:0.0:1788204904.878037:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204905.710052:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204905.710071:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204905.710182:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204905.710209:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204905.710213:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204905.710231:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204905.902093:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204906.221988:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204906.221999:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204906.222003:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204906.222015:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4037 00000100:00080000:1.0:1788204906.222027:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204906.222035:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204906.222086:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064560001280/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204906.222128:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204906.222133:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204906.222135:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204906.222140:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4037 00000100:00080000:1.0:1788204906.222144:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204906.222150:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204906.222164:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204906.222182:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064560001408/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204906.222191:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560001280:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204906.222197:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204906.222203:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204906.222205:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204906.222212:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4037 00000100:00080000:1.0:1788204906.222217:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204906.222222:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204906.222229:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204906.222244:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064560001536/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204906.222285:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204906.222342:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204906/real 1788204906] req@ffff93c1c5037100 x1875064560001280/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204906.222364:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064560001280/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204906.222377:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204906.222382:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204906.222387:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560001280:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204906.222426:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560001408:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204906.222453:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204906.222483:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204906/real 1788204906] req@ffff93c1c5034a80 x1875064560001408/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204906.222511:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064560001408/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204906.222526:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204906.222531:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204906.222535:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560001408:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204906.222542:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560001536:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204906.222566:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204906.222579:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204906/real 1788204906] req@ffff93c1c5034e00 x1875064560001536/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204906.222593:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064560001536/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204906.222604:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204906.222608:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204906.222612:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560001536:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204906.926034:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204907.950174:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204908.974013:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204909.998020:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204910.830069:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204910.830089:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204910.830220:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204910.830254:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204910.830258:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204910.830281:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204911.021893:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204911.341967:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204911.341979:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204911.341984:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204911.341994:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4042 00000100:00080000:1.0:1788204911.342008:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204911.342017:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204911.342073:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064560001664/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204911.342109:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204911.342114:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204911.342116:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204911.342121:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4042 00000100:00080000:1.0:1788204911.342124:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204911.342127:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204911.342137:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204911.342150:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064560001792/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204911.342159:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204911.342161:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204911.342163:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204911.342170:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4042 00000100:00080000:1.0:1788204911.342173:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204911.342175:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204911.342180:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204911.342186:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064560001920/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204911.342211:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560001664:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788204911.342288:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204911.342330:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204911/real 1788204911] req@ffff93c1c5035c00 x1875064560001664/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204911.342345:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064560001664/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204911.342359:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204911.342362:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204911.342366:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560001664:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204911.342371:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560001792:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204911.342386:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204911.342393:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204911/real 1788204911] req@ffff93c1c5037800 x1875064560001792/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204911.342423:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064560001792/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204911.342434:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204911.342436:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204911.342439:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560001792:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204911.342461:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560001920:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204911.342478:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204911.342486:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204911/real 1788204911] req@ffff93c1c5036680 x1875064560001920/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204911.342494:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064560001920/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204911.342502:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204911.342504:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204911.342506:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560001920:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204912.046048:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204913.069954:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204914.094068:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204915.118037:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204915.950080:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204915.950103:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204915.950240:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204915.950269:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204915.950273:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204915.950300:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204916.142040:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204916.461937:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204916.461945:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204916.461949:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204916.461956:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4047 00000100:00080000:3.0:1788204916.461965:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204916.461970:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204916.461999:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f800 x1875064560002048/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204916.462021:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204916.462023:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204916.462024:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204916.462027:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4047 00000100:00080000:3.0:1788204916.462029:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204916.462031:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204916.462038:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204916.462047:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4d180 x1875064560002176/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204916.462053:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204916.462055:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204916.462056:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204916.462058:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4047 00000100:00080000:3.0:1788204916.462060:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204916.462062:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204916.462065:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788204916.462071:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c380 x1875064560002304/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204916.462118:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560002048:192.168.201.156@tcp:250:kworker.0 00000800:00000100:0.0:1788204916.462212:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204916.462278:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204916/real 1788204916] req@ffff93c1c4a4f800 x1875064560002048/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204916.462307:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f800 x1875064560002048/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204916.462334:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204916.462338:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204916.462341:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560002048:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204916.462347:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560002176:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204916.462379:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204916.462394:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204916/real 1788204916] req@ffff93c1c4a4d180 x1875064560002176/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204916.462406:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4d180 x1875064560002176/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204916.462414:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204916.462423:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204916.462425:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560002176:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204916.462431:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560002304:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204916.462449:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204916.462457:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204916/real 1788204916] req@ffff93c1c4a4c380 x1875064560002304/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204916.462475:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c380 x1875064560002304/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204916.462499:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204916.462503:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204916.462506:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560002304:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204917.166003:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204918.190019:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204919.213992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204920.238023:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204921.070057:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204921.070079:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204921.070219:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204921.070278:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204921.070283:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204921.070313:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204921.262009:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204921.581983:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204921.581997:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204921.582002:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204921.582014:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4052 00000100:00080000:3.0:1788204921.582028:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204921.582036:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204921.582073:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e300 x1875064560002432/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204921.582103:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204921.582105:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204921.582107:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204921.582110:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4052 00000100:00080000:3.0:1788204921.582112:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204921.582114:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204921.582123:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204921.582131:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c700 x1875064560002560/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204921.582138:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204921.582141:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204921.582142:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204921.582145:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4052 00000100:00080000:3.0:1788204921.582147:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204921.582149:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204921.582152:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204921.582159:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560002432:192.168.201.156@tcp:250:kworker.0 00000100:00100000:3.0:1788204921.582160:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f480 x1875064560002688/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204921.582248:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204921.582291:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204921/real 1788204921] req@ffff93c1c4a4e300 x1875064560002432/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204921.582305:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e300 x1875064560002432/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204921.582317:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204921.582320:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204921.582323:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560002432:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204921.582329:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560002560:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204921.582344:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204921.582352:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204921/real 1788204921] req@ffff93c1c4a4c700 x1875064560002560/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204921.582359:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c700 x1875064560002560/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204921.582364:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204921.582366:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204921.582369:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560002560:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204921.582393:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560002688:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204921.582413:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204921.582425:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204921/real 1788204921] req@ffff93c1c4a4f480 x1875064560002688/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204921.582436:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f480 x1875064560002688/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204921.582446:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204921.582449:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204921.582453:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560002688:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204922.288442:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204923.310052:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204924.333996:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204925.357977:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204926.190028:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204926.190044:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204926.190182:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204926.190211:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204926.190214:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204926.190234:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204926.382015:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204926.701956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204926.701968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204926.701973:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204926.701987:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4057 00000100:00080000:1.0:1788204926.701999:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204926.702005:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204926.702047:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064560002816/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204926.702074:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204926.702076:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204926.702078:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204926.702081:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4057 00000100:00080000:1.0:1788204926.702083:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204926.702086:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204926.702095:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204926.702105:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064560002944/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204926.702113:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204926.702115:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204926.702117:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204926.702120:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560002816:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204926.702123:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4057 00000100:00080000:1.0:1788204926.702129:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204926.702133:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204926.702146:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204926.702160:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064560003072/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204926.702198:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204926.702246:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204926/real 1788204926] req@ffff93c1c5036680 x1875064560002816/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204926.702276:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064560002816/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204926.702288:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204926.702292:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204926.702296:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560002816:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204926.702319:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560002944:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204926.702336:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204926.702345:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204926/real 1788204926] req@ffff93c1c5037800 x1875064560002944/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204926.702353:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064560002944/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204926.702359:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204926.702361:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204926.702363:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560002944:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204926.702366:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560003072:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204926.702376:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204926.702382:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204926/real 1788204926] req@ffff93c1c5035c00 x1875064560003072/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204926.702389:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064560003072/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204926.702400:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204926.702402:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204926.702404:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560003072:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204927.406023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204928.430010:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204929.454031:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204930.477997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204931.310048:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204931.310082:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204931.310192:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204931.310216:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204931.310219:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204931.310234:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204931.501985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204931.821953:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204931.821964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204931.821967:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204931.821977:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4062 00000100:00080000:1.0:1788204931.821989:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204931.821995:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204931.822039:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064560003200/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204931.822081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204931.822084:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204931.822085:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204931.822088:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4062 00000100:00080000:1.0:1788204931.822091:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204931.822094:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204931.822104:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204931.822114:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064560003328/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204931.822121:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204931.822124:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204931.822126:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788204931.822130:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560003200:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204931.822132:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4062 00000100:00080000:1.0:1788204931.822137:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204931.822140:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204931.822143:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204931.822151:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064560003456/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204931.822215:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204931.822268:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204931/real 1788204931] req@ffff93c1c5034e00 x1875064560003200/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204931.822283:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064560003200/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204931.822291:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204931.822295:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204931.822298:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560003200:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204931.822319:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560003328:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204931.822335:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204931.822344:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204931/real 1788204931] req@ffff93c1c5034a80 x1875064560003328/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204931.822356:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064560003328/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204931.822370:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204931.822373:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204931.822377:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560003328:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204931.822383:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560003456:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204931.822400:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204931.822408:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204931/real 1788204931] req@ffff93c1c5037100 x1875064560003456/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204931.822415:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064560003456/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204931.822423:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204931.822425:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204931.822427:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560003456:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204932.526053:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204933.550047:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204934.574042:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204935.598002:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204936.430027:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204936.430043:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204936.430157:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204936.430178:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204936.430181:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204936.430199:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204936.621994:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204936.941984:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204936.941994:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204936.941998:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204936.942008:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4067 00000100:00080000:1.0:1788204936.942021:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204936.942027:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204936.942071:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064560003584/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204936.942096:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204936.942099:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204936.942100:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204936.942103:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4067 00000100:00080000:1.0:1788204936.942105:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204936.942108:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204936.942116:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204936.942125:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064560003712/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204936.942134:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204936.942137:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204936.942138:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204936.942142:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4067 00000100:00080000:1.0:1788204936.942144:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204936.942148:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204936.942149:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560003584:192.168.201.156@tcp:250:kworker.0 00000008:00100000:1.0:1788204936.942151:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204936.942161:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064560003840/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204936.942215:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204936.942270:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204936/real 1788204936] req@ffff93c1c5037100 x1875064560003584/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204936.942286:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064560003584/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204936.942295:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204936.942298:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204936.942301:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560003584:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204936.942321:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560003712:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204936.942336:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204936.942344:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204936/real 1788204936] req@ffff93c1c5034a80 x1875064560003712/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204936.942352:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064560003712/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204936.942358:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204936.942360:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204936.942361:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560003712:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204936.942365:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560003840:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204936.942376:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204936.942381:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204936/real 1788204936] req@ffff93c1c5034e00 x1875064560003840/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204936.942389:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064560003840/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204936.942400:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204936.942402:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204936.942404:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560003840:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204937.646017:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204938.670034:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204939.694027:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204940.717997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204941.550009:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204941.550025:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204941.550121:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204941.550143:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204941.550146:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204941.550161:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204941.742005:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204942.061986:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204942.061996:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204942.062000:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204942.062010:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4072 00000100:00080000:1.0:1788204942.062021:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204942.062027:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204942.062073:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064560003968/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204942.062104:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204942.062107:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204942.062109:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204942.062113:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4072 00000100:00080000:1.0:1788204942.062116:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204942.062120:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204942.062131:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204942.062145:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064560004096/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204942.062150:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560003968:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204942.062164:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204942.062168:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204942.062170:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204942.062174:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4072 00000100:00080000:1.0:1788204942.062185:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204942.062191:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204942.062197:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204942.062210:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064560004224/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204942.062221:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204942.062267:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204942/real 1788204942] req@ffff93c1c5035c00 x1875064560003968/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204942.062289:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064560003968/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204942.062305:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204942.062309:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204942.062313:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560003968:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204942.062340:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560004096:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204942.062364:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204942.062377:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204942/real 1788204942] req@ffff93c1c5037800 x1875064560004096/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204942.062389:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064560004096/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204942.062404:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204942.062407:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204942.062410:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560004096:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204942.062416:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560004224:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204942.062434:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204942.062443:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204942/real 1788204942] req@ffff93c1c5036680 x1875064560004224/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204942.062450:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064560004224/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204942.062464:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204942.062466:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204942.062468:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560004224:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204942.765979:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204943.789998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204944.814028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204945.838039:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788204946.669971:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204946.669984:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204946.670064:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788204946.670081:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788204946.670084:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788204946.670099:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204946.862013:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204947.181905:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204947.181913:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788204947.181916:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204947.181923:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4078 00000100:00080000:3.0:1788204947.181932:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204947.181937:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788204947.181969:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ea00 x1875064560004352/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204947.181993:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204947.181997:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788204947.181998:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204947.182003:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4078 00000100:00080000:3.0:1788204947.182006:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204947.182010:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204947.182019:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204947.182033:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4c000 x1875064560004480/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204947.182038:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560004352:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788204947.182053:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204947.182058:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788204947.182060:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788204947.182065:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4078 00000100:00080000:3.0:1788204947.182069:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788204947.182073:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788204947.182079:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788204947.182091:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4fb80 x1875064560004608/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204947.182105:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204947.182149:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204947/real 1788204947] req@ffff93c1c4a4ea00 x1875064560004352/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204947.182164:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ea00 x1875064560004352/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204947.182171:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204947.182174:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204947.182177:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560004352:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204947.182194:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560004480:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204947.182209:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204947.182217:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204947/real 1788204947] req@ffff93c1c4a4c000 x1875064560004480/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204947.182224:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4c000 x1875064560004480/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204947.182229:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204947.182231:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204947.182233:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560004480:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204947.182236:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560004608:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204947.182245:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204947.182253:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204947/real 1788204947] req@ffff93c1c4a4fb80 x1875064560004608/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204947.182263:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4fb80 x1875064560004608/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204947.182272:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204947.182275:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204947.182279:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560004608:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204947.886019:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204948.910017:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204949.934008:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204950.958061:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204951.790035:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204951.790053:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204951.790155:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204951.790187:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204951.790190:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204951.790210:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204951.982065:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204952.301965:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204952.301972:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204952.301975:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204952.301983:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4083 00000100:00080000:1.0:1788204952.301993:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204952.301999:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204952.302036:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064560004736/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204952.302063:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204952.302065:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204952.302067:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204952.302069:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4083 00000100:00080000:1.0:1788204952.302072:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204952.302075:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204952.302082:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204952.302091:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064560004864/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204952.302101:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204952.302103:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204952.302105:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204952.302107:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4083 00000100:00080000:1.0:1788204952.302109:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204952.302111:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204952.302114:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204952.302122:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560004736:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204952.302122:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064560004992/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204952.302180:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204952.302220:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204952/real 1788204952] req@ffff93c1c5036680 x1875064560004736/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204952.302232:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064560004736/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204952.302239:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204952.302242:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204952.302245:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560004736:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204952.302252:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560004864:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204952.302265:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204952.302276:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204952/real 1788204952] req@ffff93c1c5037800 x1875064560004864/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204952.302286:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064560004864/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204952.302294:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204952.302297:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204952.302300:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560004864:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204952.302322:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560004992:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204952.302337:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204952.302354:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204952/real 1788204952] req@ffff93c1c5035c00 x1875064560004992/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204952.302361:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064560004992/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204952.302368:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204952.302369:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204952.302371:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560004992:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204953.005998:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204954.030068:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204955.054036:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204956.078038:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204956.910022:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204956.910040:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204956.910156:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204956.910180:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204956.910183:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204956.910201:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204957.102029:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204957.421900:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204957.421914:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204957.421919:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204957.421929:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4088 00000100:00080000:1.0:1788204957.421941:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204957.421950:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204957.421997:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064560005120/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204957.422027:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204957.422030:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204957.422031:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204957.422034:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4088 00000100:00080000:1.0:1788204957.422037:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204957.422039:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204957.422048:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204957.422058:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064560005248/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204957.422065:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204957.422067:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204957.422068:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204957.422071:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4088 00000100:00080000:1.0:1788204957.422073:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204957.422075:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204957.422078:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204957.422082:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560005120:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204957.422087:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064560005376/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204957.422162:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204957.422212:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204957/real 1788204957] req@ffff93c1c5034e00 x1875064560005120/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204957.422229:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064560005120/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204957.422239:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204957.422242:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204957.422246:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560005120:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204957.422253:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560005248:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204957.422273:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204957.422284:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204957/real 1788204957] req@ffff93c1c5034a80 x1875064560005248/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204957.422300:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064560005248/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204957.422309:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204957.422312:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204957.422314:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560005248:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204957.422342:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560005376:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204957.422364:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204957.422374:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204957/real 1788204957] req@ffff93c1c5037100 x1875064560005376/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204957.422385:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064560005376/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204957.422395:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204957.422399:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204957.422401:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560005376:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204958.126048:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204959.150043:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204960.174023:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204961.198017:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204962.029992:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204962.030006:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204962.030117:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204962.030147:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204962.030152:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204962.030182:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204962.221956:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204962.541935:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204962.541943:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204962.541947:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204962.541954:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4093 00000100:00080000:1.0:1788204962.541963:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204962.541967:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204962.542003:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037100 x1875064560005504/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204962.542025:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204962.542028:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204962.542029:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204962.542032:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4093 00000100:00080000:1.0:1788204962.542034:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204962.542037:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204962.542043:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204962.542051:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034a80 x1875064560005632/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204962.542058:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204962.542060:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204962.542062:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204962.542064:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4093 00000100:00080000:1.0:1788204962.542066:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788204962.542074:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560005504:192.168.201.156@tcp:250:kworker.0 00000100:00080000:1.0:1788204962.542075:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204962.542082:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204962.542092:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5034e00 x1875064560005760/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204962.542134:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204962.542172:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204962/real 1788204962] req@ffff93c1c5037100 x1875064560005504/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204962.542189:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037100 x1875064560005504/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204962.542196:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204962.542199:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204962.542201:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560005504:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204962.542207:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560005632:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204962.542219:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204962.542226:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204962/real 1788204962] req@ffff93c1c5034a80 x1875064560005632/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204962.542233:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034a80 x1875064560005632/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204962.542237:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204962.542239:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204962.542240:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560005632:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204962.542256:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560005760:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204962.542266:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204962.542271:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204962/real 1788204962] req@ffff93c1c5034e00 x1875064560005760/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204962.542278:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5034e00 x1875064560005760/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204962.542288:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204962.542290:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204962.542291:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5034e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560005760:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204963.246014:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204964.270034:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204965.294055:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204966.318002:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204967.150059:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204967.150076:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204967.150175:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204967.150191:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204967.150194:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204967.150208:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204967.342012:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204967.661997:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204967.662005:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204967.662008:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204967.662016:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4098 00000100:00080000:0.0:1788204967.662026:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204967.662031:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204967.662061:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064560005888/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204967.662084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204967.662086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204967.662087:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204967.662090:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4098 00000100:00080000:0.0:1788204967.662092:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204967.662094:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204967.662102:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204967.662111:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064560006016/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204967.662118:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204967.662120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204967.662121:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204967.662125:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4098 00000100:00080000:0.0:1788204967.662128:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204967.662130:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204967.662133:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204967.662145:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064560006144/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204967.662148:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560005888:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204967.662272:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560006016:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204967.662327:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560006144:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204968.365965:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204969.389954:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204970.413974:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788204970.734070:0:2330:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.156:988 00000400:00000100:3.0:1788204970.734106:0:2330:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.156@tcp at host 192.168.201.156 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788204970.734119:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204970.734164:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000800:00000100:3.0:1788204970.734173:0:2330:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.156@tcp 00000100:00100000:1.0:1788204970.734224:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204967/real 1788204970] req@ffff93c0e9ce5880 x1875064560006144/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204970.734263:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064560006144/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204970.734277:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204970.734283:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204970.734286:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560006144:192.168.201.156@tcp:8:kworker.0 00000100:00100000:1.0:1788204970.734294:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204967/real 1788204970] req@ffff93c0e9ce6300 x1875064560005888/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204970.734307:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064560005888/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204970.734314:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204970.734317:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204970.734320:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560005888:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204970.734327:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204967/real 1788204970] req@ffff93c0e9ce4e00 x1875064560006016/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204970.734350:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064560006016/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204970.734358:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204970.734361:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204970.734363:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560006016:192.168.201.156@tcp:38:kworker.0 00010000:00010000:2.0:1788204971.438045:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204972.270005:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204972.270026:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204972.270143:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204972.270168:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204972.270172:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204972.270192:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204972.461990:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204972.781953:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204972.781964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204972.781968:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204972.781979:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4103 00000100:00080000:0.0:1788204972.781990:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204972.781996:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204972.782037:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4e00 x1875064560006272/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204972.782064:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204972.782066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204972.782067:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204972.782070:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4103 00000100:00080000:0.0:1788204972.782073:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204972.782075:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204972.782092:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204972.782107:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce6300 x1875064560006400/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788204972.782114:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560006272:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204972.782124:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204972.782136:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788204972.782137:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204972.782143:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4103 00000100:00080000:0.0:1788204972.782146:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204972.782149:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204972.782154:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204972.782172:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5880 x1875064560006528/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204972.782190:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204972.782252:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204972/real 1788204972] req@ffff93c0e9ce4e00 x1875064560006272/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204972.782271:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4e00 x1875064560006272/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204972.782284:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204972.782288:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204972.782293:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560006272:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204972.782329:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560006400:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204972.782366:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204972.782379:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204972/real 1788204972] req@ffff93c0e9ce6300 x1875064560006400/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204972.782392:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce6300 x1875064560006400/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204972.782419:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204972.782422:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204972.782425:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560006400:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204972.782432:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560006528:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204972.782452:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204972.782464:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204972/real 1788204972] req@ffff93c0e9ce5880 x1875064560006528/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204972.782474:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5880 x1875064560006528/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204972.782484:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204972.782487:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204972.782490:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560006528:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204973.485997:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204974.509961:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204975.534047:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204976.557991:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204977.390049:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204977.390069:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204977.390190:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204977.390212:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204977.390215:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204977.390231:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204977.581967:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204977.901976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204977.901990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204977.901994:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204977.902007:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4108 00000100:00080000:2.0:1788204977.902020:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204977.902027:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204977.902069:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9500 x1875064560006656/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204977.902093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204977.902096:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204977.902097:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204977.902100:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4108 00000100:00080000:2.0:1788204977.902103:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204977.902105:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204977.902115:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204977.902124:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb800 x1875064560006784/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204977.902137:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788204977.902138:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560006656:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204977.902140:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204977.902143:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204977.902150:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4108 00000100:00080000:2.0:1788204977.902152:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204977.902154:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204977.902157:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204977.902168:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8a80 x1875064560006912/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204977.902215:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204977.902271:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204977/real 1788204977] req@ffff93c0e62b9500 x1875064560006656/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204977.902291:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9500 x1875064560006656/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204977.902301:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204977.902305:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204977.902309:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560006656:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204977.902335:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560006784:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204977.902354:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204977.902364:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204977/real 1788204977] req@ffff93c0e62bb800 x1875064560006784/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204977.902375:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb800 x1875064560006784/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204977.902383:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204977.902386:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204977.902389:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560006784:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204977.902394:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560006912:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204977.902408:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204977.902415:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204977/real 1788204977] req@ffff93c0e62b8a80 x1875064560006912/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204977.902425:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8a80 x1875064560006912/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204977.902435:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204977.902439:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204977.902441:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560006912:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204978.605985:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204979.629995:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204980.653967:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204981.677987:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788204982.510037:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204982.510053:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204982.510163:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788204982.510245:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788204982.510248:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788204982.510268:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204982.701973:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204983.021914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204983.021928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788204983.021933:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204983.021943:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4113 00000100:00080000:0.0:1788204983.021955:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204983.021963:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204983.022000:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7800 x1875064560007040/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204983.022025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204983.022028:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788204983.022029:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204983.022033:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4113 00000100:00080000:0.0:1788204983.022035:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204983.022037:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204983.022046:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204983.022055:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5500 x1875064560007168/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788204983.022063:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204983.022066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1788204983.022066:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560007040:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788204983.022070:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788204983.022076:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4113 00000100:00080000:0.0:1788204983.022078:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788204983.022081:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788204983.022084:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788204983.022092:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5180 x1875064560007296/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204983.022134:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204983.022183:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204983/real 1788204983] req@ffff93c0e9ce7800 x1875064560007040/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204983.022198:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7800 x1875064560007040/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204983.022206:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204983.022209:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204983.022212:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560007040:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204983.022234:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560007168:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788204983.022256:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204983.022268:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204983/real 1788204983] req@ffff93c0e9ce5500 x1875064560007168/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204983.022279:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5500 x1875064560007168/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204983.022288:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204983.022291:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204983.022294:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560007168:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788204983.022300:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560007296:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788204983.022321:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788204983.022331:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204983/real 1788204983] req@ffff93c0e9ce5180 x1875064560007296/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204983.022341:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5180 x1875064560007296/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204983.022355:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204983.022357:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788204983.022361:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560007296:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204983.726001:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204984.749995:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204985.774023:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204986.797951:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204987.629971:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204987.629986:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204987.630103:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204987.630124:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204987.630127:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204987.630145:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788204987.821999:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204988.141982:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204988.141990:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204988.141993:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204988.142001:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4119 00000100:00080000:1.0:1788204988.142009:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204988.142015:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204988.142044:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064560007424/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204988.142105:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204988.142109:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204988.142110:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204988.142113:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4119 00000100:00080000:1.0:1788204988.142116:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204988.142121:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788204988.142124:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560007424:192.168.201.156@tcp:250:kworker.0 00000008:00100000:1.0:1788204988.142130:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788204988.142142:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064560007552/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204988.142149:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204988.142152:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204988.142154:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204988.142158:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4119 00000100:00080000:1.0:1788204988.142161:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204988.142165:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204988.142169:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204988.142181:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064560007680/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204988.142196:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204988.142242:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204988/real 1788204988] req@ffff93c1c5035c00 x1875064560007424/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204988.142255:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064560007424/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204988.142263:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204988.142266:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204988.142270:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560007424:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204988.142287:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560007552:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204988.142304:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204988.142315:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204988/real 1788204988] req@ffff93c1c5037800 x1875064560007552/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204988.142327:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064560007552/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204988.142341:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204988.142344:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204988.142347:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560007552:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204988.142353:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560007680:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204988.142375:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204988.142382:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204988/real 1788204988] req@ffff93c1c5036680 x1875064560007680/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204988.142389:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064560007680/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204988.142395:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204988.142397:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204988.142399:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560007680:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204988.846025:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204989.870032:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204990.893954:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788204991.918068:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788204992.749950:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204992.749962:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204992.750083:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788204992.750106:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788204992.750109:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788204992.750125:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788204992.941968:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204993.261935:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204993.261945:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788204993.261949:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204993.261958:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4124 00000100:00080000:1.0:1788204993.261969:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204993.261975:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:1.0:1788204993.262022:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5036680 x1875064560007808/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204993.262044:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204993.262047:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788204993.262049:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204993.262053:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4124 00000100:00080000:1.0:1788204993.262057:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204993.262061:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204993.262069:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788204993.262074:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560007808:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788204993.262082:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5037800 x1875064560007936/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788204993.262090:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204993.262093:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788204993.262095:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788204993.262098:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4124 00000100:00080000:1.0:1788204993.262101:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788204993.262104:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:1.0:1788204993.262108:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788204993.262116:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c5035c00 x1875064560008064/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204993.262133:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204993.262168:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204993/real 1788204993] req@ffff93c1c5036680 x1875064560007808/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204993.262181:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5036680 x1875064560007808/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204993.262191:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204993.262194:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204993.262196:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5036680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560007808:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204993.262213:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560007936:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204993.262229:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204993.262236:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204993/real 1788204993] req@ffff93c1c5037800 x1875064560007936/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204993.262243:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5037800 x1875064560007936/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204993.262248:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204993.262250:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204993.262252:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5037800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560007936:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204993.262255:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560008064:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204993.262265:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204993.262270:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204993/real 1788204993] req@ffff93c1c5035c00 x1875064560008064/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204993.262276:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c5035c00 x1875064560008064/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204993.262282:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204993.262283:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204993.262285:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c5035c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560008064:192.168.201.156@tcp:8:kworker.0 00010000:00010000:2.0:1788204993.965938:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204994.989951:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788204996.014017:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788204997.037982:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788204997.869981:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204997.869993:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204997.870081:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788204997.870104:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788204997.870108:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788204997.870128:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788204998.062009:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204998.381995:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204998.382006:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788204998.382011:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204998.382020:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4129 00000100:00080000:2.0:1788204998.382031:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204998.382038:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788204998.382075:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9c00 x1875064560008192/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788204998.382110:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204998.382113:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788204998.382115:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204998.382119:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4129 00000100:00080000:2.0:1788204998.382123:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204998.382127:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204998.382135:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788204998.382149:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b9f80 x1875064560008320/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788204998.382161:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560008192:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788204998.382161:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204998.382171:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788204998.382173:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788204998.382178:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4129 00000100:00080000:2.0:1788204998.382181:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788204998.382185:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788204998.382190:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788204998.382209:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bad80 x1875064560008448/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788204998.382223:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204998.382262:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204998/real 1788204998] req@ffff93c0e62b9c00 x1875064560008192/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204998.382275:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9c00 x1875064560008192/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204998.382283:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204998.382285:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204998.382287:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560008192:192.168.201.156@tcp:250:kworker.0 00000100:00100000:0.0:1788204998.382303:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560008320:192.168.201.156@tcp:38:kworker.0 00000800:00000100:0.0:1788204998.382316:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204998.382323:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204998/real 1788204998] req@ffff93c0e62b9f80 x1875064560008320/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204998.382330:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b9f80 x1875064560008320/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204998.382335:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204998.382336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204998.382338:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560008320:192.168.201.156@tcp:38:kworker.0 00000100:00100000:0.0:1788204998.382341:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560008448:192.168.201.156@tcp:8:kworker.0 00000800:00000100:0.0:1788204998.382352:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:0.0:1788204998.382357:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204998/real 1788204998] req@ffff93c0e62bad80 x1875064560008448/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204998.382363:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bad80 x1875064560008448/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788204998.382369:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204998.382370:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:0.0:1788204998.382372:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560008448:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788204999.085928:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788205000.110028:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205001.133944:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788205002.158514:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788205002.989992:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788205002.990007:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788205002.990096:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788205002.990117:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788205002.990122:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788205002.990151:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788205003.181960:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205003.501938:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205003.501946:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788205003.501949:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205003.501957:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4134 00000100:00080000:0.0:1788205003.501965:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205003.501970:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:0.0:1788205003.502000:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce4a80 x1875064560008576/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205003.502021:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205003.502023:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788205003.502024:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205003.502027:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4134 00000100:00080000:0.0:1788205003.502029:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205003.502031:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788205003.502046:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788205003.502055:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce7100 x1875064560008704/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788205003.502062:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205003.502065:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788205003.502066:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788205003.502070:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4134 00000100:00100000:1.0:1788205003.502071:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560008576:192.168.201.156@tcp:250:kworker.0 00000100:00080000:0.0:1788205003.502072:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788205003.502078:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:0.0:1788205003.502083:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:0.0:1788205003.502090:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e9ce5f80 x1875064560008832/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788205003.502137:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205003.502173:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205003/real 1788205003] req@ffff93c0e9ce4a80 x1875064560008576/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205003.502190:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce4a80 x1875064560008576/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205003.502197:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205003.502199:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205003.502202:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560008576:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788205003.502219:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560008704:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788205003.502234:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205003.502242:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205003/real 1788205003] req@ffff93c0e9ce7100 x1875064560008704/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205003.502249:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce7100 x1875064560008704/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205003.502254:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205003.502256:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205003.502258:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560008704:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788205003.502261:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560008832:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788205003.502271:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205003.502276:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205003/real 1788205003] req@ffff93c0e9ce5f80 x1875064560008832/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205003.502287:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e9ce5f80 x1875064560008832/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205003.502293:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205003.502296:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205003.502299:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e9ce5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560008832:192.168.201.156@tcp:8:kworker.0 00010000:00010000:1.0:1788205004.206008:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205005.230014:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788205006.254017:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205007.277979:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788205008.110045:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788205008.110064:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788205008.110200:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788205008.110231:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788205008.110235:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788205008.110262:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788205008.301977:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205008.621968:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205008.621978:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788205008.621983:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205008.621993:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4139 00000100:00080000:2.0:1788205008.622004:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205008.622011:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788205008.622052:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bbb80 x1875064560008960/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205008.622077:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205008.622080:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788205008.622081:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205008.622084:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4139 00000100:00080000:2.0:1788205008.622086:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205008.622089:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788205008.622098:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205008.622105:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb100 x1875064560009088/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205008.622111:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205008.622114:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788205008.622115:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205008.622119:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4139 00000100:00080000:2.0:1788205008.622120:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205008.622123:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788205008.622133:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788205008.622142:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560008960:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788205008.622144:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8700 x1875064560009216/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788205008.622206:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205008.622250:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205008/real 1788205008] req@ffff93c0e62bbb80 x1875064560008960/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205008.622265:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bbb80 x1875064560008960/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205008.622273:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205008.622276:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205008.622278:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560008960:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788205008.622283:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560009088:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788205008.622297:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205008.622304:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205008/real 1788205008] req@ffff93c0e62bb100 x1875064560009088/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205008.622316:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb100 x1875064560009088/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205008.622321:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205008.622323:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205008.622325:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560009088:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788205008.622352:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560009216:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788205008.622364:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205008.622369:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205008/real 1788205008] req@ffff93c0e62b8700 x1875064560009216/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205008.622377:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8700 x1875064560009216/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205008.622383:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205008.622385:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205008.622387:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560009216:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788205009.326010:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788205010.350052:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205011.374002:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788205012.398046:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1788205013.230029:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788205013.230047:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788205013.230165:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1788205013.230192:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1788205013.230197:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1788205013.230224:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788205013.422030:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205013.741994:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205013.742007:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788205013.742012:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205013.742025:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4144 00000100:00080000:2.0:1788205013.742038:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205013.742053:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788205013.742093:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8000 x1875064560009344/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205013.742121:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205013.742123:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788205013.742124:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205013.742128:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4144 00000100:00080000:2.0:1788205013.742130:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205013.742133:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788205013.742143:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205013.742152:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62bb480 x1875064560009472/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205013.742160:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788205013.742161:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560009344:192.168.201.156@tcp:250:kworker.0 00000100:00080000:2.0:1788205013.742165:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788205013.742169:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205013.742173:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4144 00000100:00080000:2.0:1788205013.742176:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205013.742178:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788205013.742182:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:2.0:1788205013.742189:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064560009600/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788205013.742236:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205013.742290:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205013/real 1788205013] req@ffff93c0e62b8000 x1875064560009344/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205013.742305:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8000 x1875064560009344/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205013.742313:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205013.742317:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205013.742320:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560009344:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788205013.742342:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560009472:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788205013.742357:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205013.742366:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205013/real 1788205013] req@ffff93c0e62bb480 x1875064560009472/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205013.742374:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62bb480 x1875064560009472/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205013.742379:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205013.742381:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205013.742383:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560009472:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788205013.742386:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560009600:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788205013.742396:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205013.742402:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205013/real 1788205013] req@ffff93c0e62b8380 x1875064560009600/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205013.742408:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064560009600/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205013.742415:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205013.742417:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205013.742418:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560009600:192.168.201.156@tcp:8:kworker.0 00010000:00010000:1.0:1788205014.446007:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205015.469973:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788205016.493963:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205017.518019:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788205018.350066:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788205018.350080:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788205018.350185:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788205018.350212:0:18723:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788205018.350217:0:18723:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788205018.350245:0:18723:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788205018.542005:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205018.861953:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205018.861964:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788205018.861972:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205018.861983:0:18723:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4149 00000100:00080000:2.0:1788205018.861993:0:18723:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205018.862000:0:18723:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:2.0:1788205018.862038:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62ba680 x1875064560009728/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205018.862066:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205018.862070:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788205018.862071:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205018.862075:0:18723:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4149 00000100:00080000:2.0:1788205018.862079:0:18723:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205018.862083:0:18723:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788205018.862093:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788205018.862107:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62baa00 x1875064560009856/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788205018.862117:0:18723:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205018.862122:0:18723:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788205018.862124:0:18723:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788205018.862129:0:18723:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4149 00000100:00080000:2.0:1788205018.862133:0:18723:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788205018.862138:0:18723:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:2.0:1788205018.862143:0:18723:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:1.0:1788205018.862159:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560009728:192.168.201.156@tcp:250:kworker.0 00000100:00100000:2.0:1788205018.862159:0:18723:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c0e62b8380 x1875064560009984/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788205018.862260:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205018.862323:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205018/real 1788205018] req@ffff93c0e62ba680 x1875064560009728/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205018.862345:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62ba680 x1875064560009728/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205018.862357:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205018.862361:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205018.862365:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560009728:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788205018.862374:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560009856:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788205018.862395:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205018.862407:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205018/real 1788205018] req@ffff93c0e62baa00 x1875064560009856/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205018.862418:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62baa00 x1875064560009856/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205018.862425:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205018.862428:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205018.862431:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560009856:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788205018.862459:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560009984:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788205018.862480:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205018.862490:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205018/real 1788205018] req@ffff93c0e62b8380 x1875064560009984/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205018.862501:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c0e62b8380 x1875064560009984/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205018.862511:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205018.862514:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205018.862517:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c0e62b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560009984:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788205019.565995:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788205020.589985:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205021.614004:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788205022.638033:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788205023.470069:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788205023.470088:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788205023.470210:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788205023.470241:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788205023.470245:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788205023.470274:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1788205023.661992:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205023.981995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205023.982009:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788205023.982014:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205023.982026:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4154 00000100:00080000:3.0:1788205023.982039:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205023.982045:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788205023.982090:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4ed80 x1875064560010112/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205023.982115:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205023.982119:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788205023.982121:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205023.982125:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4154 00000100:00080000:3.0:1788205023.982128:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205023.982133:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788205023.982144:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205023.982159:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4dc00 x1875064560010240/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205023.982167:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560010112:192.168.201.156@tcp:250:kworker.0 00000100:00080000:3.0:1788205023.982174:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205023.982178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788205023.982181:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205023.982186:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4154 00000100:00080000:3.0:1788205023.982191:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205023.982194:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788205023.982198:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788205023.982209:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064560010368/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788205023.982245:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205023.982304:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205023/real 1788205023] req@ffff93c1c4a4ed80 x1875064560010112/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205023.982322:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4ed80 x1875064560010112/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205023.982332:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205023.982336:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205023.982340:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560010112:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788205023.982364:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560010240:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788205023.982383:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205023.982393:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205023/real 1788205023] req@ffff93c1c4a4dc00 x1875064560010240/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205023.982405:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4dc00 x1875064560010240/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205023.982412:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205023.982415:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205023.982418:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560010240:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788205023.982424:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560010368:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788205023.982441:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205023.982450:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205023/real 1788205023] req@ffff93c1c4a4e680 x1875064560010368/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205023.982475:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064560010368/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205023.982486:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205023.982488:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205023.982491:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560010368:192.168.201.156@tcp:8:kworker.0 00010000:00010000:1.0:1788205024.686030:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205025.709952:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788205026.734047:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205027.758019:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788205028.590046:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788205028.590065:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788205028.590188:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788205028.590218:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788205028.590221:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788205028.590272:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1788205028.782050:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205029.101948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 961d4966-9b2e-4bca-bd35-8d9493e230e8->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205029.101960:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788205029.101964:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205029.101973:0:37:0:(import.c:510:import_select_connection()) MGC192.168.201.156@tcp: connect to NID 192.168.201.156@tcp last attempt 4159 00000100:00080000:3.0:1788205029.101984:0:37:0:(import.c:589:import_select_connection()) MGC192.168.201.156@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205029.101991:0:37:0:(import.c:624:import_select_connection()) MGC192.168.201.156@tcp: import ffff93c1c85d5000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000100:00100000:3.0:1788205029.102036:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4df80 x1875064560010496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205029.102066:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205029.102069:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788205029.102071:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205029.102075:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4159 00000100:00080000:3.0:1788205029.102079:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205029.102082:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff93c1d85e6000: import ffff93c1c4d85000 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788205029.102092:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682934 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205029.102104:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4f100 x1875064560010624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205029.102116:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 986744c8-ee81-4a47-8d91-97f81ede1f3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205029.102119:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788205029.102121:0:37:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788205029.102126:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: connect to NID 192.168.201.156@tcp last attempt 4159 00000100:00080000:3.0:1788205029.102130:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788205029.102133:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff93c1d85e6000: import ffff93c1d3241800 using connection 192.168.201.156@tcp/192.168.201.156@tcp 00000008:00100000:3.0:1788205029.102139:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682934 ocd_grant: 487325696, lost: 0. 00000100:00100000:3.0:1788205029.102151:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff93c1c4a4e680 x1875064560010752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205029.102161:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560010496:192.168.201.156@tcp:250:kworker.0 00000800:00000100:1.0:1788205029.102243:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205029.102287:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205029/real 1788205029] req@ffff93c1c4a4df80 x1875064560010496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205029.102307:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4df80 x1875064560010496/t0(0) o250->MGC192.168.201.156@tcp@192.168.201.156@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205029.102315:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c85d5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205029.102318:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205029.102321:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:961d4966-9b2e-4bca-bd35-8d9493e230e8:2340:1875064560010496:192.168.201.156@tcp:250:kworker.0 00000100:00100000:1.0:1788205029.102327:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560010624:192.168.201.156@tcp:38:kworker.0 00000800:00000100:1.0:1788205029.102355:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205029.102364:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205029/real 1788205029] req@ffff93c1c4a4f100 x1875064560010624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205029.102387:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4f100 x1875064560010624/t0(0) o38->lustre-MDT0000-mdc-ffff93c1d85e6000@192.168.201.156@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205029.102396:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1c4d85000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205029.102398:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205029.102400:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560010624:192.168.201.156@tcp:38:kworker.0 00000100:00100000:1.0:1788205029.102425:0:2340:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560010752:192.168.201.156@tcp:8:kworker.0 00000800:00000100:1.0:1788205029.102439:0:2340:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.156@tcp 00000100:00100000:1.0:1788205029.102447:0:2340:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205029/real 1788205029] req@ffff93c1c4a4e680 x1875064560010752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205029.102454:0:2340:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff93c1c4a4e680 x1875064560010752/t0(0) o8->lustre-OST0000-osc-ffff93c1d85e6000@192.168.201.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205029.102466:0:2340:0:(import.c:65:import_set_state_nolock()) ffff93c1d3241800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205029.102468:0:2340:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.156@tcp failed (-110) 00000100:00100000:1.0:1788205029.102477:0:2340:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff93c1c4a4e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:986744c8-ee81-4a47-8d91-97f81ede1f3e:2340:1875064560010752:192.168.201.156@tcp:8:kworker.0 00010000:00010000:0.0:1788205029.805998:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788205030.830038:0:10052:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1788205031.854042:0:10053:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 82854 lines, 82854 kept, 0 dropped, 0 bad.